FEAT: Add conda packaging recipes and publish tooling for mssql-python

.github/workflows/conda-audit.yml

@@ -0,0 +1,85 @@
+---
+# N3: PR-triggered conda build + masking-immune RUNPATH audit.
+#
+# The OneBranch conda-build pipeline is `trigger: none` / `pr: none`, so its blocking
+# audit never runs on a PR. This lightweight GitHub Actions job builds ONE real
+# linux-64 conda package from the SHIPPED PyPI wheels and runs
+# eng/scripts/audit_bundled_binaries.py on it, so a regression in the $ORIGIN climb,
+# the declared conda deps (krb5/libtool/openssl), or the expected DT_NEEDED set fails
+# the PR automatically -- the full-agent runtime masking cannot hide it.
+#
+# The PyPI mssql-python-odbc binaries are not yet pre-baked with the climb, so the
+# recipe's assertion-only default would (correctly) refuse to mutate them; this PR
+# gate sets CONDA_ALLOW_UNSIGNED_PATCH=1 to build a DEV-patched climb and audit THAT.
+# The signed release path stays assertion-only (see conda/mssql-python/build.sh).
+name: conda-audit
+
+on:
+  pull_request:
+    paths:
+      - 'conda/**'
+      - 'eng/scripts/audit_bundled_binaries.py'
+      - 'OneBranchPipelines/scripts/build-conda-packages.sh'
+      - 'tests/test_027_conda_release_metadata.py'
+      - 'tests/test_029_bundled_binary_audit.py'
+      - '.github/workflows/conda-audit.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  linux-conda-audit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Unit-test the audit + release validator
+        run: |
+          python -m pip install --quiet pytest zstandard
+          # --noconftest: tests/conftest.py imports mssql_python (the native ddbc_bindings
+          # extension), which is NOT built in this repackage-only gate. test_027/test_029
+          # are pure conda validator/audit tests using only built-in fixtures, so skip
+          # conftest to avoid that unrelated import.
+          python -m pytest --noconftest \
+                           tests/test_029_bundled_binary_audit.py \
+                           tests/test_027_conda_release_metadata.py -q
+
+      - name: Fetch the shipped linux wheels from PyPI
+        run: |
+          set -euo pipefail
+          mkdir -p wheels
+          # --no-deps: pull ONLY the two wheels we repackage (not azure-identity etc).
+          python -m pip download --no-deps mssql-python mssql-python-odbc -d wheels
+          echo "Downloaded:"; ls -1 wheels
+
+      - name: Build + audit a linux-64 conda package
+        env:
+          # PyPI odbc binaries are not pre-baked; build a DEV-patched climb here and
+          # audit it. NEVER used on the signed release path.
+          CONDA_ALLOW_UNSIGNED_PATCH: '1'
+        run: |
+          set -euo pipefail
+          mssql_whl="$(ls wheels/mssql_python-*.whl | grep -v mssql_python_odbc | head -1)"
+          odbc_whl="$(ls wheels/mssql_python_odbc-*.whl | head -1)"
+          MSSQL_VER="$(basename "$mssql_whl" | sed -nE 's/^mssql_python-([^-]+)-.*/\1/p')"
+          ODBC_VER="$(basename "$odbc_whl" | sed -nE 's/^mssql_python_odbc-([^-]+)-.*/\1/p')"
+          echo "mssql-python=$MSSQL_VER  mssql-python-odbc=$ODBC_VER"
+          bash OneBranchPipelines/scripts/build-conda-packages.sh \
+            "$PWD/wheels" \
+            "$PWD/conda" \
+            "$RUNNER_TEMP/conda-bld" \
+            "$MSSQL_VER" \
+            "$ODBC_VER" \
+            "3.11" \
+            "linux-64"
+
+      - name: Explicit standalone audit of the built package
+        run: |
+          set -euo pipefail
+          python -m pip install --quiet zstandard
+          python eng/scripts/audit_bundled_binaries.py --root "$RUNNER_TEMP/conda-bld/bld"

OneBranchPipelines/conda-release-pipeline.yml

@@ -0,0 +1,146 @@
+# =========================================================================================
+# OneBranch STANDALONE Conda Release Pipeline (mssql-python)
+# =========================================================================================
+# Decoupled from the wheel/PyPI release. Downloads the consolidated conda packages produced
+# by the STANDALONE conda-build pipeline (conda-build-pipeline.yml, artifact
+# drop_ConsolidateConda_ConsolidateArtifacts) -- NOT the wheel pipeline (def 2199), which no
+# longer produces any conda artifact -- enforces the release-readiness gate (conda-release-
+# step: required subdirs + full Python matrix + one version), and -- when publishToConda=true -- publishes the SELF-CONTAINED
+# mssql-python conda package to Anaconda.org via anaconda-client (conda-publish-step; ESRP
+# has no Conda ContentType). There is a single self-contained package (it vendors the ODBC
+# Driver 18 payload); no companion.
+#
+# Always Official (a publish is a deliberate, gated action). Manual trigger only.
+# The OneBranch YAML needs an actual ADO run to fully validate.
+# =========================================================================================
+name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaRelease
+
+trigger: none
+pr: none
+
+parameters:
+  - name: publishToConda
+    displayName: 'Publish Conda Packages to Anaconda.org (PRODUCTION)'
+    type: boolean
+    default: false  # Safety: default to a validate-only dry run.
+  - name: condaChannel
+    displayName: 'Anaconda.org channel/org to publish to'
+    type: string
+    default: 'microsoft'
+  - name: condaLabel
+    displayName: 'Anaconda.org channel label'
+    type: string
+    default: 'main'
+  # Infra setup: ADO definition id of the STANDALONE conda-build pipeline
+  # (conda-build-pipeline.yml), whose ConsolidateConda stage produces
+  # drop_ConsolidateConda_ConsolidateArtifacts. This is NOT the wheel pipeline
+  # (def 2199). Set it once the conda-build pipeline is registered in ADO; 0 is a
+  # placeholder that must be overridden before a real run.
+  - name: condaBuildDefinitionId
+    displayName: 'Conda-build pipeline ADO definition id'
+    type: number
+    default: 0
+  # The exact mssql-python version being released. When set, the readiness gate
+  # asserts EVERY conda package matches it (not just internal one-version
+  # consistency). Leave empty only for a consistency-only dry run.
+  - name: mssqlPythonVersion
+    displayName: 'Expected mssql-python release version (e.g. 1.13.0)'
+    type: string
+    default: ''
+
+variables:
+  - template: /OneBranchPipelines/variables/common-variables.yml@self
+  - template: /OneBranchPipelines/variables/onebranch-variables.yml@self
+  # ANACONDA_API_TOKEN lives in this org-scoped group; included ONLY when publishing so a
+  # validate-only run never requires the group to exist.
+  - ${{ if eq(parameters.publishToConda, true) }}:
+      - group: 'Anaconda Publishing'
+
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: 'OneBranch.Pipelines/GovernedTemplates'
+      ref: 'refs/heads/main'
+  # The build pipeline whose consolidated conda artifact this pipeline validates + publishes.
+  pipelines:
+    - pipeline: buildPipeline
+      # Infra setup: the STANDALONE conda-build pipeline (backed by
+      # conda-build-pipeline.yml), whose ConsolidateConda stage publishes
+      # drop_ConsolidateConda_ConsolidateArtifacts. Update to its exact ADO
+      # definition name once registered. NOT the wheel pipeline
+      # (Build-Release-Package-Pipeline / def 2199), which no longer builds conda.
+      source: 'Conda-Build-Pipeline'
+      trigger: none
+
+extends:
+  template: 'v2/OneBranch.Official.CrossPlat.yml@templates'
+  parameters:
+    featureFlags:
+      WindowsHostVersion:
+        Version: '2022'
+    globalSdl:
+      baseline:
+        baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines
+        suppressionSet: default
+      suppression:
+        suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress
+        suppressionSet: default
+      binskim:
+        enabled: true
+        break: true
+      credscan:
+        enabled: true
+      policheck:
+        enabled: true
+        break: true
+        exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml'
+      publishLogs:
+        enabled: true
+      tsa:
+        enabled: true
+        configFile: '$(REPO_ROOT)/.config/tsaoptions.json'
+
+    stages:
+      - stage: CondaRelease
+        displayName: 'Validate & Publish Conda Release'
+        jobs:
+          # Gate: prove the consolidated conda set is complete (required subdirs +
+          # full Python matrix + one version) before anything is published.
+          - job: ValidateConda
+            displayName: 'Validate consolidated conda packages'
+            pool:
+              type: windows
+              isCustom: true
+              name: Python-1ES-pool
+              demands:
+              - imageOverride -equals PYTHON-1ES-MMS2022
+            variables:
+              ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+            steps:
+              - template: /OneBranchPipelines/steps/conda-release-step.yml@self
+                parameters:
+                  buildDefinitionId: ${{ parameters.condaBuildDefinitionId }}
+                  mssqlPythonVersion: '${{ parameters.mssqlPythonVersion }}'
+
+          # PRODUCTION publish (releaseJob) -- runs ONLY when publishToConda=true and
+          # ONLY after the ValidateConda gate succeeds, so an incomplete set is never
+          # uploaded. anaconda-client reads ANACONDA_API_TOKEN from the env (never the
+          # command line), so the token never appears in the logs.
+          - ${{ if eq(parameters.publishToConda, true) }}:
+            - job: PublishConda
+              displayName: 'Publish conda packages to Anaconda.org'
+              dependsOn: ValidateConda
+              templateContext:
+                type: releaseJob
+                isProduction: true
+              pool:
+                type: windows
+              variables:
+                ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+              steps:
+                - template: /OneBranchPipelines/steps/conda-publish-step.yml@self
+                  parameters:
+                    buildDefinitionId: ${{ parameters.condaBuildDefinitionId }}
+                    condaChannel: '${{ parameters.condaChannel }}'
+                    condaLabel: '${{ parameters.condaLabel }}'

OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml

@@ -0,0 +1,126 @@
+# Consolidate Conda Artifacts Job Template
+# Collects the per-platform self-contained mssql-python conda packages (which vendor
+# the ODBC payload) that each build leg staged under conda/<subdir>/ and gathers
+# them into a single conda/ tree for the release pipeline to publish.
+#
+# BEST-EFFORT (build pipeline): conda is a downstream repackage of the ESRP-signed
+# wheels and must NEVER block the primary wheel deliverable, so a missing/short set
+# only WARNS here. The HARD count gate lives in the release pipeline (which refuses
+# to publish an incomplete conda set), symmetric with how the wheel/odbc drops are
+# best-effort collected in the build and gated at release time.
+#
+# Expected packages (validated conda subdirs). The self-contained mssql-python
+# package (which vendors the ODBC payload) is emitted per-Python by each build leg;
+# there is NO separate companion package:
+#   win-64        : 5 py x mssql-python                                       = 5
+#   osx-64        : 5 py x mssql-python  (Intel Mac, cross-built via Rosetta) = 5
+#   osx-arm64     : 5 py x mssql-python  (Apple Silicon, native)              = 5
+#   linux-64      : 5 py x mssql-python  (glibc x86_64 host, native)          = 5
+#   linux-aarch64 : 5 py x mssql-python  (x86_64 host + QEMU, best-effort)    = 5
+#   ------------------------------------------------------------------------------
+#   TOTAL (PyPI parity minus win-arm64 + musllinux)                          = 25
+# win-arm64 (no import-validation host on x64) and musllinux (no conda musl subdir)
+# are intentionally NOT conda-built. This job is BEST-EFFORT and never hard-fails on
+# a short set; the release pipeline's conda-release-step enforces the hard gate
+# (required subdirs present + complete Python matrix) before anything is published.
+parameters:
+  - name: oneBranchType
+    type: string
+    default: 'Official'
+  # Artifact item pattern the consolidate job downloads. Defaults to the integrated
+  # wheel-pipeline leg artifacts; the standalone conda-build pipeline overrides it with
+  # its drop_Conda* leg artifacts.
+  - name: downloadItemPattern
+    type: string
+    default: |
+      drop_Win_*/**
+      drop_MacOS_*/**
+      drop_Linux_*/**
+      drop_ODBC_BuildAll_*/**
+
+jobs:
+  - job: ConsolidateArtifacts
+    displayName: 'Consolidate All Conda Packages'
+    condition: succeeded()
+
+    pool:
+      type: linux
+      isCustom: true
+      name: Azure Pipelines
+      vmImage: 'ubuntu-latest'
+
+    variables:
+      # Consolidation only moves files; no binaries to scan.
+      - name: ob_sdl_binskim_enabled
+        value: false
+      - name: ob_outputDirectory
+        value: '$(Build.ArtifactStagingDirectory)'
+
+    steps:
+      - checkout: self
+        fetchDepth: 1
+
+      # The conda packages are staged INSIDE the mssql-python build-leg artifacts
+      # (drop_Win_*, drop_MacOS_*, drop_Linux_*) under conda/<subdir>/. Scope the
+      # download to those stages so every leg's self-contained mssql-python conda is
+      # gathered in one place. (drop_ODBC_BuildAll_* is included only for its wheels,
+      # which ride along and are ignored below -- we pick only *.conda / *.tar.bz2.)
+      - task: DownloadPipelineArtifact@2
+        displayName: 'Download All Platform Artifacts'
+        inputs:
+          buildType: 'current'
+          itemPattern: ${{ parameters.downloadItemPattern }}
+          targetPath: '$(Pipeline.Workspace)/all-artifacts'
+
+      - bash: |
+          set -e
+          echo "Collecting conda packages (preserving <subdir>/ layout)..."
+          mkdir -p $(ob_outputDirectory)/conda
+
+          # Copy every mssql-python* conda package into conda/<subdir>/. Each build
+          # leg wrote the self-contained mssql-python package under a conda/<subdir>/
+          # folder, so the parent dir name IS the target subdir.
+          found=0
+          while IFS= read -r p; do
+            subdir=$(basename "$(dirname "$p")")
+            mkdir -p "$(ob_outputDirectory)/conda/$subdir"
+            cp -v "$p" "$(ob_outputDirectory)/conda/$subdir/"
+            found=1
+          done < <(find $(Pipeline.Workspace)/all-artifacts -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \))
+
+          echo ""
+          echo "Consolidated conda tree:"
+          find $(ob_outputDirectory)/conda -type f | sort
+
+          PKG_COUNT=$(find $(ob_outputDirectory)/conda -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l)
+          echo ""
+          echo "Per-subdir conda package counts:"
+          for d in $(ob_outputDirectory)/conda/*/; do
+            [ -d "$d" ] || continue
+            sub=$(basename "$d")
+            n=$(find "$d" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l)
+            printf '  %-14s %s\n' "$sub" "$n"
+          done
+          echo "Total conda package count: $PKG_COUNT (full PyPI-parity set = 25)"
+
+          # BEST-EFFORT: warn only, never exit non-zero — a conda hiccup on any leg
+          # must not fail this build or block the wheel release. The release pipeline's
+          # conda-release-step enforces the hard gate (required subdirs + full Python
+          # matrix) before anything is published.
+          if [ "$found" != "1" ]; then
+            echo "##vso[task.logissue type=warning]No conda packages found in the build-leg artifacts."
+          else
+            echo "Collected $PKG_COUNT conda package(s) (best-effort; release-time gate enforces completeness)."
+          fi
+        displayName: 'Consolidate conda packages'
+
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish Consolidated Conda Artifacts'
+        inputs:
+          targetPath: '$(ob_outputDirectory)'
+          # Distinct name so it does not collide with the wheel consolidate artifact
+          # (drop_Consolidate_ConsolidateArtifacts) or the odbc one
+          # (drop_ConsolidateOdbc_ConsolidateArtifacts) in the same run. Matches the
+          # OneBranch auto-name for a stage named `ConsolidateConda`.
+          artifact: 'drop_ConsolidateConda_ConsolidateArtifacts'
+          publishLocation: 'pipeline'

OneBranchPipelines/scripts/.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf