diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..5e76f91 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: Run nightly package version and status checks + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + actions: write + issues: read + +env: + PIP_EXTRA_INDEX_URL: https://gitlab.com/api/v4/projects/56254198/packages/pypi/simple + +jobs: + check_versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3' + + - name: Install Python dependencies + run: pip install requests packaging + + - name: Check versions and open PRs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 ci_scripts/check_versions.py --summary --create-prs + + check_deprecated_packages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install curl and jq + run: sudo apt-get update -qq && sudo apt-get install -qq -y curl jq + + - name: Verify riscv64 wheels still published + run: | + ret=0 + while IFS= read -r pkg; do + [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue + + if ! output=$(curl -sf "https://pypi.org/pypi/${pkg}/json"); then + echo "Failed to fetch $pkg, skipping..." + continue + fi + + if ! filenames=$(echo "$output" | jq -r '.urls[].filename'); then + echo "Failed to parse JSON for $pkg, skipping..." + continue + fi + + if ! echo "$filenames" | grep -qi riscv; then + echo "$pkg does not distribute riscv64 wheel anymore!" + ret=1 + fi + done < ci_scripts/deprecated.txt + exit $ret + + check_installation: + runs-on: ubuntu-24.04-riscv + steps: + - uses: actions/checkout@v4 + + - name: Install packages inside riscv64 container + working-directory: ci_scripts + run: | + sed -Ei 's/^scipy.*$//' packages.txt + sed -Ei 's/^patchelf.*$//' packages.txt + ./list-packages.sh python-wheels + + pip_audit: + runs-on: ubuntu-24.04-riscv + continue-on-error: true + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + + - name: Run pip-audit + working-directory: ci_scripts + run: ./audit.sh python-wheels + + - name: Report vulnerabilities and open issues + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 ci_scripts/audit_report.py ci_scripts/audit-report.json diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..93c0fde --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: PR verification checks + +on: + pull_request: + +jobs: + check_commit_messages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Reject revertme / DO NOT MERGE commits + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + range="$BASE_SHA..$HEAD_SHA" + if git log --pretty=format:%s "$range" | grep -i -e '^revertme' -e '^revert me'; then + echo "::error::Merge request contains at least a commit starting with a 'revert me' tag. Fix it before merging." + exit 1 + fi + if git log --pretty=format:%s "$range" | grep -i -e '^DO NOT MERGE'; then + echo "::error::Merge request contains at least a commit starting with a 'DO NOT MERGE' tag. Review it." + exit 1 + fi + + check_patches: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3' + + - name: Validate Upstream-Status in added/modified patches + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python ci_scripts/check_patch.py "$BASE_SHA" "$HEAD_SHA" diff --git a/.github/workflows/pr-trigger.yml b/.github/workflows/pr-trigger.yml new file mode 100644 index 0000000..ebb5507 --- /dev/null +++ b/.github/workflows/pr-trigger.yml @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: PR package build trigger + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + +permissions: + contents: read + actions: write + pull-requests: read + +jobs: + dispatch-triggers: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.body, 'Trigger:') + steps: + - uses: actions/checkout@v4 + with: + package_version: ${{ github.event.pull_request.head.sha }} + + - name: Dispatch build/test workflows for Trigger directives + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_BODY: ${{ github.event.pull_request.body }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPO: ${{ github.repository }} + run: | + set -euo pipefail + + if [ "$HEAD_REPO" != "$BASE_REPO" ]; then + echo "PR from fork ($HEAD_REPO); cannot dispatch workflows against fork ref. Skipping." + exit 0 + fi + + triggers=$(printf '%s\n' "$PR_BODY" | grep -oE '^Trigger:[[:space:]]*[^[:space:]]+' || true) + + if [ -z "$triggers" ]; then + echo "No Trigger: directives found." + exit 0 + fi + + printf '%s\n' "$triggers" | while IFS= read -r line; do + directive=${line#Trigger:} + directive=${directive# } + package_name=${directive%:*} + package_version=${directive#*:} + + if [ -z "$package_name" ] || [ -z "$package_version" ] || [ "$package_name" = "$package_version" ]; then + echo "Skipping malformed directive: $line" + continue + fi + + # Normalize: build workflows prepend `v` to the version themselves + # (e.g. `ref: v${{ env.NUMPY_VERSION }}`). Strip any leading v/V + # in the directive so `Trigger: numpy:v2.5.1` and + # `Trigger: numpy:2.5.1` behave identically. + package_version=${package_version#[vV]} + + build_wf=".github/workflows/build-${package_name}.yml" + test_wf=".github/workflows/test-${package_name}.yml" + + if [ ! -f "$build_wf" ]; then + echo "No build workflow for $package_name at $build_wf; skipping." + continue + fi + + echo "Dispatching build-${package_name}.yml on $HEAD_REF with version=$package_version" + gh workflow run "build-${package_name}.yml" --ref "$HEAD_REF" -f "version=$package_version" + + if [ -f "$test_wf" ]; then + echo "Dispatching test-${package_name}.yml on $HEAD_REF with version=$package_version" + gh workflow run "test-${package_name}.yml" --ref "$HEAD_REF" -f "version=$package_version" + fi + done diff --git a/LICENSE-Apache-2.0 b/LICENSE-Apache-2.0 new file mode 100644 index 0000000..45cf1ee --- /dev/null +++ b/LICENSE-Apache-2.0 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Rivos Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ci_scripts/audit.sh b/ci_scripts/audit.sh new file mode 100755 index 0000000..3b15a67 --- /dev/null +++ b/ci_scripts/audit.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: Apache-2.0 + +# This script is based on the one located here: +# https://gitlab.com/riseproject/python/wheel_builder/-/blob/main/ci_scripts/audit.sh +# It is designed to run pip-audit against the list of packages built by RISE to +# identify any vulnerabilities that may need to be addressed. You can view the +# PyPI page for more details: https://pypi.org/project/pip-audit/ +set -e + +apt-get update -qq && apt-get install -qq -y python3 python3-venv +python3 -m venv packages-$1-venv +. packages-$1-venv/bin/activate +python -m pip install --upgrade pip +python -m pip install pip-audit +python -m pip_audit -r packages.txt --format json --output audit-report.json || true + diff --git a/ci_scripts/audit_report.py b/ci_scripts/audit_report.py new file mode 100644 index 0000000..2409a69 --- /dev/null +++ b/ci_scripts/audit_report.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +""" +Parse pip-audit JSON report, emit warning annotations, append a job summary, +and open/update GitHub issues (one per (package, vulnerability id)). + +Expects environment: + GH_TOKEN authenticated gh token + GITHUB_REPOSITORY owner/repo (auto-set on GitHub Actions) + GITHUB_SERVER_URL e.g. https://github.com (auto-set) + GITHUB_RUN_ID current workflow run id (auto-set) + GITHUB_STEP_SUMMARY path to write step summary (auto-set) + +Usage: audit_report.py +""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Optional + +ISSUE_LABELS = ["pip-audit", "security"] + + +def run_url() -> str: + server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repo = os.environ.get("GITHUB_REPOSITORY", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + if not repo or not run_id: + return "" + return f"{server}/{repo}/actions/runs/{run_id}" + + +def append_summary(text: str) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + with open(path, "a") as f: + f.write(text) + + +def gh(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["gh", *args], capture_output=True, text=True, check=check + ) + + +def find_existing_issue(title: str) -> Optional[int]: + """Return open issue number matching exact title with the pip-audit label, else None.""" + try: + result = gh( + "issue", "list", + "--label", "pip-audit", + "--state", "open", + "--search", f'"{title}" in:title', + "--json", "number,title", + "--limit", "50", + ) + except subprocess.CalledProcessError as e: + print(f" [!] gh issue list failed: {e.stderr or e}", file=sys.stderr) + return None + + try: + issues = json.loads(result.stdout) + except json.JSONDecodeError: + return None + + for issue in issues: + if issue.get("title") == title: + return issue.get("number") + return None + + +def create_issue(title: str, body: str) -> Optional[int]: + args = ["issue", "create", "--title", title, "--body", body] + for label in ISSUE_LABELS: + args += ["--label", label] + try: + result = gh(*args) + except subprocess.CalledProcessError as e: + print(f" [!] gh issue create failed: {e.stderr or e}", file=sys.stderr) + return None + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("http") and "/issues/" in line: + try: + return int(line.rsplit("/", 1)[-1]) + except ValueError: + pass + return None + + +def comment_issue(number: int, body: str) -> bool: + try: + gh("issue", "comment", str(number), "--body", body) + return True + except subprocess.CalledProcessError as e: + print(f" [!] gh issue comment failed: {e.stderr or e}", file=sys.stderr) + return False + + +def format_body(pkg: str, version: str, vuln: Dict, url: str) -> str: + fix_versions = vuln.get("fix_versions") or [] + aliases = vuln.get("aliases") or [] + description = (vuln.get("description") or "").strip() + + lines = [ + f"`pip-audit` detected a vulnerability in **{pkg}** {version}.", + "", + f"- Vulnerability ID: `{vuln.get('id', 'unknown')}`", + ] + if aliases: + lines.append(f"- Aliases: {', '.join(f'`{a}`' for a in aliases)}") + if fix_versions: + lines.append(f"- Fix versions: {', '.join(f'`{v}`' for v in fix_versions)}") + else: + lines.append("- Fix versions: _none reported_") + lines += [ + f"- First observed: {url}", + "", + "### Description", + "", + description or "_(no description provided)_", + ] + return "\n".join(lines) + + +def format_comment(pkg: str, version: str, url: str) -> str: + return ( + f"Still detected in {url}\n" + f"- Package: `{pkg}` {version}\n" + ) + + +def process_report(report_path: Path) -> int: + if not report_path.exists(): + print(f"[!] Report not found at {report_path}; nothing to process.") + return 0 + + try: + data = json.loads(report_path.read_text()) + except json.JSONDecodeError as e: + print(f"[!] Failed to parse {report_path}: {e}") + return 0 + + deps = data.get("dependencies", []) + url = run_url() + vuln_count = 0 + summary_rows = [] + + for dep in deps: + pkg = dep.get("name") + version = dep.get("version", "") + vulns = dep.get("vulns") or [] + if not pkg or not vulns: + continue + + for vuln in vulns: + vuln_id = vuln.get("id", "unknown") + title = f"pip-audit: {pkg} — {vuln_id}" + fix_versions = vuln.get("fix_versions") or [] + fixes = ", ".join(fix_versions) if fix_versions else "none" + + # GHA warning annotation (yellow bubble on job) + print( + f"::warning file=ci_scripts/packages.txt::" + f"{pkg} {version}: {vuln_id} (fix: {fixes})" + ) + + summary_rows.append( + f"| `{pkg}` | {version} | `{vuln_id}` | {fixes} |" + ) + + existing = find_existing_issue(title) + if existing is not None: + if comment_issue(existing, format_comment(pkg, version, url)): + print(f" [+] Updated existing issue #{existing} for {pkg} {vuln_id}") + else: + print(f" [!] Failed to comment on issue #{existing} for {pkg} {vuln_id}") + else: + number = create_issue(title, format_body(pkg, version, vuln, url)) + if number is not None: + print(f" [+] Created issue #{number} for {pkg} {vuln_id}") + else: + print(f" [!] Failed to create issue for {pkg} {vuln_id}") + + vuln_count += 1 + + if summary_rows: + summary = ( + "## pip-audit report\n\n" + f"Detected **{vuln_count}** vulnerability(ies). " + f"Run: {url}\n\n" + "| Package | Version | Vulnerability | Fix versions |\n" + "|---------|---------|---------------|--------------|\n" + + "\n".join(summary_rows) + + "\n" + ) + else: + summary = f"## pip-audit report\n\nNo vulnerabilities detected. Run: {url}\n" + + append_summary(summary) + print(f"[+] Processed {vuln_count} vulnerability(ies).") + return 0 + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: audit_report.py ", file=sys.stderr) + return 2 + return process_report(Path(sys.argv[1])) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci_scripts/check_patch.py b/ci_scripts/check_patch.py new file mode 100644 index 0000000..cd3f4ee --- /dev/null +++ b/ci_scripts/check_patch.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: 2025 BayLibre, SAS +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: Apache-2.0 +""" +This script is based on the one located at: + +https://gitlab.com/riseproject/python/wheel_builder/-/blob/main/ci_scripts/check_patch.py + +Its purpose is to check the contents of patches (commits) submitted to the +project for aan "Upstream-Status" tag, which is inspired by the same tag used +in the Yocto Project: + +https://docs.yoctoproject.org/dev/contributor-guide/recipe-style-guide.html#patch-upstream-status + +See the valid_statuses dictionary below for types of Upstream-Status tags +accepted. +""" + +import sys +import subprocess +import re + +valid_statuses = { + "Issue": { + "value": r"^\[https?://\S+\]$", # Mandatory url in brackets + "hint": "[url]", + "mandatory": True, + "description": "Brackets enclosed url of the issue opened on upstream project", + }, + "Submitted": { + "value": r"^\[https?://\S+\]$", # Mandatory url in brackets + "hint": "[url]", + "mandatory": True, + "description": "Brackets enclosed url of the merge request opened on upstream project", + }, + "To upstream": { + "value": r"^(\[.*\])?$", # Optional comment in brackets + "hint": "[comment]", + "mandatory": False, + "description": "Optional brackets enclosed comment to add any useful information for future upstream submission", + }, + "Inappropriate": { + "value": r"^\[.+\]$", # Any comment in brackets + "hint": "[comment]", + "mandatory": True, + "description": "Brackets enclosed reason why the patch is inappropriate for upstream", + }, + "Backport": { + "value": r"^\[https?://\S+\]$", # Mandatory url in brackets + "hint": "[url]", + "mandatory": True, + "description": "Brackets enclosed url of the backported patch", + }, +} + + +def get_commits_between(start_ref, end_ref): + """Get commit hashes between two branches or commits.""" + cmd = ["git", "rev-list", f"{start_ref}..{end_ref}"] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout.splitlines() + + +def get_commit_files(commit): + """Get a list of patch files added or modified in a commit.""" + cmd = ["git", "diff-tree", "--no-commit-id", "--name-status", "-r", commit] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + files = [] + for line in result.stdout.splitlines(): + status, filename = line.split("\t", 1) + if status in ("A", "M") and filename.endswith(".patch"): + files.append(filename) + return files + + +def extract_patch_from_commit(commit, patch_file): + """Extract a patch file from a given commit.""" + cmd = ["git", "show", f"{commit}:{patch_file}"] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: Could not extract {patch_file} from commit {commit}") + return None + return result.stdout + + +def check_upstream_status(patch_content, patch): + """Check if the Upstream-Status tag is correctly formatted in a patch file.""" + + status_key = "|".join(valid_statuses.keys()) + match = re.search(r"^Upstream-Status: *(.*)$", patch_content, re.MULTILINE) + + if not match: + print(f"❌Missing Upstream-Status tag in {patch}") + return False + + value = match.groups() + + match = re.search(rf"^({status_key}|\w+) *(.*)$", value[0], re.MULTILINE) + + status, extra = match.groups() + + if status not in valid_statuses.keys(): + print(f"❌Invalid Upstream-Status '{status}' in {patch}") + return False + + if not re.match(valid_statuses[status]["value"], extra): + print(f"❌Incorrect format for Upstream-Status '{status}' in {patch}") + return False + + return True + + +def main(): + if len(sys.argv) != 3: + print("Usage: python script.py ") + sys.exit(1) + + start_ref, end_ref = sys.argv[1], sys.argv[2] + commits = get_commits_between(start_ref, end_ref) + + error_found = False + + for commit in commits: + patch_files = get_commit_files(commit) + for patch in patch_files: + patch_content = extract_patch_from_commit(commit, patch) + if patch_content: + if not check_upstream_status(patch_content, patch): + error_found = True + else: + print(f"✅{patch}") + + if error_found: + print("Valid formats:") + for key, value in valid_statuses.items(): + print(f" - Upstream-Status: {key} {value["hint"]}") + print( + f" {value["hint"]}: {value["description"]} {'(mandatory)' if value["mandatory"] else ''}" + ) + sys.exit(1) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py new file mode 100755 index 0000000..5ab3456 --- /dev/null +++ b/ci_scripts/check_versions.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: MIT +""" +Script to check if packages in the riscv64 registry are up to date with PyPI. + +This script compares versions between the riscv64 registry and PyPI to determine +if packages should be upgraded or can be deprecated. + +It is inspired by the check_versions.py script at: + +https://gitlab.com/riseproject/python/wheel_builder/-/blob/main/ci_scripts/check_versions.py +""" + +import re +import subprocess +import sys +import os +from typing import Dict, List, Optional +from packaging import version +from pathlib import Path +import requests + + +REGISTRY_URL = "https://gitlab.com/api/v4/projects/56254198/packages/pypi/simple" +PACKAGES_FILE = "ci_scripts/packages.txt" +REPO = "riseproject-dev/python-wheels" + +# Cap on how many new versions get dispatched per upgrade PR. When the registry +# drifts far behind PyPI, an unbounded loop could kick off dozens of workflow +# runs per package. Three keeps the retry cost bounded while still covering the +# common "we missed one or two point releases" case. +MAX_NEW_VERSIONS_PER_PR = 3 + + +def read_packages() -> List[str]: + """Read the list of packages from packages.txt.""" + packages = [] + try: + with open(PACKAGES_FILE, 'r') as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + packages.append(line) + except FileNotFoundError: + print(f"Error: {PACKAGES_FILE} not found") + sys.exit(1) + return packages + + +def get_registry_latest_version(package: str) -> Optional[str]: + """Get the latest version available in the riscv64 registry.""" + try: + result = subprocess.run([ + "pip", "index", "versions", package, + "--index-url", REGISTRY_URL, + "--platform", "manylinux_2_34_riscv64", + "--platform", "manylinux_2_35_riscv64", + "--platform", "manylinux_2_39_riscv64", + "--python-version", "3.12" + ], capture_output=True, text=True, timeout=30) + + if result.returncode != 0: + return None + + for line in result.stdout.split('\n'): + if "Available versions:" in line: + versions_part = line.split("Available versions:")[1].strip() + if versions_part: + versions = [v.strip() for v in versions_part.split(',')] + return versions[0] if versions else None + return None + except (subprocess.TimeoutExpired, subprocess.SubprocessError): + return None + + +def get_pypi_package_info(package: str) -> Optional[Dict]: + """Get package information from PyPI API.""" + try: + response = requests.get(f"https://pypi.org/pypi/{package}/json", timeout=30) + response.raise_for_status() + return response.json() + except requests.RequestException: + return None + + +def get_pypi_latest_version(package_info: Dict) -> str: + return package_info["info"]["version"] + + +def get_pypi_package_url(package_info: Dict) -> str: + return package_info["info"]["package_url"] + + +def has_riscv64_wheel(package_info: Dict, target_version: str) -> bool: + """Check if a specific version has riscv64 wheels available.""" + releases = package_info.get("releases", {}) + for release in releases.get(target_version, []): + if "riscv64" in release.get("filename", "").lower(): + return True + return False + + +def is_pure_python_wheel(package_info: Dict, target_version: str) -> bool: + """Check if a version only has pure Python wheels.""" + releases = package_info.get("releases", {}) + has_wheels = False + for release in releases.get(target_version, []): + filename = release.get("filename", "") + if filename.endswith(".whl"): + has_wheels = True + if "py3-none-any" not in filename.lower(): + return False + return has_wheels + + +def git_run(*args, subdir: str = None) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], cwd=subdir, check=True, capture_output=True, text=True) + + +def find_upstream_issue(package: str) -> Optional[str]: + """ + Find issue for a package in the wheel_builder's Upstream milestone. + + All issues in the upstream milestone follow the same title format: + + {package} riscv64 support + """ + try: + issue_title = f"{package} riscv64 support" + result = subprocess.run([ + "gh", "issue", "list", + "--milestone", "Upstream", + "--search", issue_title, + "--json", "number", + "--jq", ".[0].number", + "--repo", REPO, + ], capture_output=True, text=True, timeout=30) + + if result.returncode != 0: + return None + + return result.stdout.strip() or None + + except (subprocess.TimeoutExpired, subprocess.CalledProcessError, Exception): + return None + + +def configure_git_identity(): + git_run("config", "user.name", "github-actions[bot]") + git_run("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + + +def extract_pr_url(stdout: str) -> Optional[str]: + for line in stdout.split('\n'): + line = line.strip() + if "github.com" in line and "/pull/" in line: + return line + return None + + +def create_deprecation_pr(package: str, reason: str) -> Optional[str]: + """Create a pull request to deprecate a package.""" + try: + git_run("fetch", "origin") + git_run("switch", "main") + + yaml_file = Path(f"docs/source/packages/{package}.yaml") + if not yaml_file.exists(): + print(f" [!] YAML file for {package} does not exist") + return None + + content = yaml_file.read_text() + + if content.startswith("deprecated:"): + print(f" [!] {package} is already deprecated") + return None + + new_content = f"deprecated:\n{content}" + + upstream_issue = find_upstream_issue(package) + if upstream_issue: + print(f" [+] Found upstream issue #{upstream_issue} for {package}") + else: + print(f" [!] No upstream issue found for {package}") + + branch = f"github-actions/deprecate-{package}" + + configure_git_identity() + git_run("switch", "-c", branch) + + yaml_file.write_text(new_content) + + packages_file = Path("ci_scripts/packages.txt") + if packages_file.exists(): + lines = packages_file.read_text().splitlines() + updated_lines = [] + for line in lines: + stripped_line = line.strip() + if stripped_line and not stripped_line.startswith('#') and stripped_line == package: + continue + updated_lines.append(line) + packages_file.write_text('\n'.join(updated_lines) + '\n') + + deprecated_file = Path("ci_scripts/deprecated.txt") + lines = deprecated_file.read_text().splitlines() + + comments = lines[:7] + deprecated_packages = lines[7:] + + if package not in deprecated_packages: + deprecated_packages.append(package) + deprecated_packages.sort(key=lambda s: s.lower()) + updated_lines = comments + deprecated_packages + deprecated_file.write_text('\n'.join(updated_lines) + '\n') + + commit_title = f"{package}: deprecate our wheel" + fix_tag = f"Fixes: #{upstream_issue}\n\n" if upstream_issue else "" + + git_run("add", str(yaml_file)) + git_run("add", str(packages_file)) + git_run("add", str(deprecated_file)) + git_run("commit", "-s", "-m", f"{commit_title}\n\n{reason}\n\n{fix_tag}") + + git_run("push", "origin", branch) + + result = subprocess.run([ + "gh", "pr", "create", "--draft", + "--repo", REPO, + "--base", "main", + "--head", branch, + "--reviewer", "threexc,justeph", + "--title", f"{package}: deprecate our wheel", + "--body", f"Automatically generated PR to deprecate {package}.\n\n{reason}\n\n{fix_tag}", + ], capture_output=True, text=True, check=True) + + return extract_pr_url(result.stdout) or f"PR created for {package} (URL not found in output)" + + except subprocess.CalledProcessError as e: + print(f" [X] Error creating PR for {package}: {e.stderr or e}") + return None + except Exception as e: + print(f" [X] Unexpected error creating PR for {package}: {e}") + return None + + +def dispatch_workflow(workflow: str, ref: str, version_input: str) -> bool: + """ + Dispatch a workflow_dispatch workflow with a version input. + + Strips any leading `v`/`V` from version_input; build workflows already + prepend `v` when constructing git refs (e.g. `ref: v${{ env.VERSION }}`), + so passing `v2.5.1` would produce `vv2.5.1`. + """ + version_input = version_input.lstrip("vV") + try: + subprocess.run([ + "gh", "workflow", "run", workflow, + "--repo", REPO, + "--ref", ref, + "-f", f"version={version_input}", + ], check=True, capture_output=True, text=True) + return True + except subprocess.CalledProcessError as e: + print(f" [!] Failed to dispatch {workflow}: {e.stderr or e}") + return False + + +def get_new_versions(package_info: Dict, registry_version: str) -> List[str]: + """ + Return sorted list of stable PyPI versions strictly greater than + registry_version. Skips prereleases, dev releases, and fully yanked + versions. + + Result is truncated to the newest MAX_NEW_VERSIONS_PER_PR entries so a + stale registry does not spawn dozens of workflow dispatches per package. + Older skipped versions can still be triggered manually by editing the PR + body to add more `Trigger:` directives. + """ + try: + reg_ver = version.parse(registry_version) + except version.InvalidVersion: + return [] + + result = [] + for raw, files in (package_info.get("releases") or {}).items(): + try: + v = version.parse(raw) + except version.InvalidVersion: + continue + if v.is_prerelease or v.is_devrelease: + continue + if v <= reg_ver: + continue + if files and all(f.get("yanked") for f in files): + continue + result.append(raw) + + result.sort(key=version.parse) + return result[-MAX_NEW_VERSIONS_PER_PR:] + + +def find_workflow_default_version(content: str) -> Optional[str]: + """ + Return the value of `default:` under `workflow_dispatch.inputs.version`. + + Uses a simple heuristic: the first `default:` after `workflow_dispatch:`. + This holds for the current build-.yml template. + """ + marker = re.search(r"workflow_dispatch:", content) + if not marker: + return None + m = re.search( + r"^\s*default:\s*(['\"]?)([^'\"\n\r]+?)\1\s*$", + content[marker.end():], + re.MULTILINE, + ) + return m.group(2).strip() if m else None + + +def bump_workflow_version(path: Path, new_version: str) -> bool: + """ + Update the workflow_dispatch default version in a workflow file, and + replace every quoted literal of the old version elsewhere in the file + (concurrency group, env defaults, job names). + + Returns True if the file was modified. + """ + content = path.read_text() + current = find_workflow_default_version(content) + if current is None or current == new_version: + return False + updated = re.sub( + r"(['\"])" + re.escape(current) + r"\1", + lambda m: f"{m.group(1)}{new_version}{m.group(1)}", + content, + ) + if updated == content: + return False + path.write_text(updated) + return True + + +def create_upgrade_pr(package: str, package_info: Dict, new_versions: List[str]) -> Optional[str]: + """ + Create a PR to trigger builds for each new package version. + + - Bumps the `default:` version in build-.yml (and test-.yml if + present) to the latest new version. + - PR body includes a `Trigger: :` line for every new + version so the pr-trigger workflow (or a human editing the body) can + re-dispatch builds. + - Script also directly dispatches build-.yml (and test-.yml if + present) once per new version, because PRs authored by GITHUB_TOKEN do + not fire pull_request workflows. + - If build workflow is missing, PR body contains boilerplate asking the + reviewer to create it. + """ + if not new_versions: + return None + + latest_version = new_versions[-1] + + try: + configure_git_identity() + + git_run("fetch", "origin") + git_run("switch", "main") + + branch = f"github-actions/upgrade-{package}-{latest_version}" + git_run("switch", "-c", branch) + + pypi_package_url = get_pypi_package_url(package_info) + build_workflow = Path(f".github/workflows/build-{package}.yml") + test_workflow = Path(f".github/workflows/test-{package}.yml") + + bumped: List[Path] = [] + if build_workflow.exists() and bump_workflow_version(build_workflow, latest_version): + git_run("add", str(build_workflow)) + bumped.append(build_workflow) + if test_workflow.exists() and bump_workflow_version(test_workflow, latest_version): + git_run("add", str(test_workflow)) + bumped.append(test_workflow) + + if bumped: + commit_message = f"Upgrade {package} to v{latest_version}" + git_run("commit", "-m", commit_message) + else: + commit_message = f"DO NOT MERGE: Upgrade {package}: trigger builds for v{latest_version}" + git_run("commit", "--allow-empty", "-m", commit_message) + + git_run("push", "origin", branch) + + versions_list = ", ".join(f"v{v}" for v in new_versions) + body_lines = [ + f"Automatically generated PR to upgrade {package} — new versions detected: {versions_list}.", + "", + f"Link to [PyPI]({pypi_package_url}).", + "", + ] + + if bumped: + bumped_list = ", ".join(f"`{p}`" for p in bumped) + body_lines += [ + f"Bumped default `version` input to v{latest_version} in {bumped_list}.", + "", + ] + + if build_workflow.exists(): + body_lines += [ + "Build (and test, if present) workflows dispatched automatically for each version listed below.", + "", + "Trigger directives (re-dispatch by editing the PR body):", + "", + ] + body_lines += [f"Trigger: {package}:{v}" for v in new_versions] + body_lines += [ + "", + "If all builds succeed without modification, merge this PR and re-trigger the workflow(s) from `main` to publish.", + "If changes are needed, force-push this branch and re-dispatch.", + ] + else: + body_lines += [ + f"No build workflow found at `.github/workflows/build-{package}.yml`.", + "Please create one (and optionally a matching `test-*.yml`) before merging.", + "", + "Once the workflow exists, edit this PR body to re-fire the `Trigger:` directives:", + "", + ] + body_lines += [f"Trigger: {package}:{v}" for v in new_versions] + + body = "\n".join(body_lines) + "\n" + + result = subprocess.run([ + "gh", "pr", "create", "--draft", + "--repo", REPO, + "--base", "main", + "--head", branch, + "--reviewer", "threexc,justeph", + "--title", f"Upgrade {package} to v{latest_version}", + "--body", body, + ], capture_output=True, text=True, check=True) + + pr_url = extract_pr_url(result.stdout) or f"PR created for {package} (URL not found in output)" + + if build_workflow.exists(): + for v in new_versions: + print(f" [+] Dispatching build-{package}.yml on {branch} for v{v}") + dispatch_workflow(f"build-{package}.yml", branch, v) + if test_workflow.exists(): + print(f" [+] Dispatching test-{package}.yml on {branch} for v{v}") + dispatch_workflow(f"test-{package}.yml", branch, v) + else: + print(f" [!] No build workflow for {package}; skipping dispatch") + + return pr_url + + except subprocess.CalledProcessError as e: + print(f" [X] Error creating upgrade PR for {package}: {e.stderr or e}") + return None + except Exception as e: + print(f" [X] Unexpected error creating upgrade PR for {package}: {e}") + return None + + +def compare_versions(registry_version: str, pypi_version: str) -> int: + """Compare two version strings. Returns 0 if version matches, -1 otherwise.""" + reg_ver = version.parse(registry_version) + pypi_ver = version.parse(pypi_version) + if reg_ver < pypi_ver: + return -1 + return 0 + + +def check_package(package: str, create_prs: bool = False) -> Dict[str, any]: + """Check a single package version and optionally create PRs.""" + + registry_version = get_registry_latest_version(package) + if registry_version is None: + print(f"[X] Could not get registry version for {package}") + return {"status": "error", "package": package, "error": "Could not get registry version"} + + pypi_info = get_pypi_package_info(package) + if pypi_info is None: + print(f"[X] Could not get PyPI info for {package}") + return {"status": "error", "package": package, "error": "Could not get PyPI info"} + + pypi_version = get_pypi_latest_version(pypi_info) + pypi_package_url = get_pypi_package_url(pypi_info) + + if compare_versions(registry_version, pypi_version) == 0: + print(f"[+] {package} v{pypi_version} is up to date") + return { + "status": "up_to_date", + "package": package, + "registry_version": registry_version, + "pypi_version": pypi_version + } + + if has_riscv64_wheel(pypi_info, pypi_version): + print(f"[-] {package} v{pypi_version} has riscv64 wheels on PyPI. Can be deprecated.") + reason = f"{package} v{pypi_version} has riscv64 wheels on PyPI: {pypi_package_url}" + pr_url = create_deprecation_pr(package, reason) if create_prs else None + return { + "status": "can_deprecate", + "package": package, + "registry_version": registry_version, + "pypi_version": pypi_version, + "reason": reason, + "pr_url": pr_url, + } + + if is_pure_python_wheel(pypi_info, pypi_version): + print(f"[-] {package} v{pypi_version} switched to pure Python wheels only. Can be deprecated.") + reason = f"{package} v{pypi_version} switched to pure Python wheels only: {pypi_package_url}" + pr_url = create_deprecation_pr(package, reason) if create_prs else None + return { + "status": "can_deprecate", + "package": package, + "registry_version": registry_version, + "pypi_version": pypi_version, + "reason": reason, + "pr_url": pr_url, + } + + new_versions = get_new_versions(pypi_info, registry_version) or [pypi_version] + print(f"[^] {package} can be upgraded: v{registry_version} -> {', '.join(f'v{v}' for v in new_versions)}") + pr_url = create_upgrade_pr(package, pypi_info, new_versions) if create_prs else None + return { + "status": "need_upgrade", + "package": package, + "registry_version": registry_version, + "pypi_version": pypi_version, + "new_versions": new_versions, + "pr_url": pr_url, + } + + +def print_summary(results: List[Dict[str, any]]): + """Print a formatted summary of all results.""" + up_to_date, can_deprecate, need_upgrade, errors = [], [], [], [] + + for result in results: + { + "up_to_date": up_to_date, + "can_deprecate": can_deprecate, + "need_upgrade": need_upgrade, + "error": errors, + }.get(result["status"], []).append(result) + + print("\n" + "=" * 80) + print("SUMMARY") + print("=" * 80) + + print(f"\n[+] UP TO DATE ({len(up_to_date)} packages):") + if up_to_date: + max_name_len = max(len(r["package"]) for r in up_to_date) + for result in sorted(up_to_date, key=lambda x: x["package"]): + print(f" {result['package']:<{max_name_len}} v{result['pypi_version']}") + else: + print(" (none)") + + print(f"\n[-] CAN BE DEPRECATED ({len(can_deprecate)} packages):") + if can_deprecate: + max_name_len = max(len(r["package"]) for r in can_deprecate) + max_reg_version_len = max(len(f"v{r['registry_version']}") for r in can_deprecate) + for result in sorted(can_deprecate, key=lambda x: x["package"]): + reg_version = f"v{result['registry_version']}" + pypi_version = f"v{result['pypi_version']}" + print(f" {result['package']:<{max_name_len}} {reg_version:>{max_reg_version_len}} -> {pypi_version} ({result['reason']})") + if result.get('pr_url'): + print(f" {'':<{max_name_len}} {'':<{max_reg_version_len}} PR: {result['pr_url']}") + else: + print(" (none)") + + print(f"\n[^] NEED UPGRADE ({len(need_upgrade)} packages):") + if need_upgrade: + max_name_len = max(len(r["package"]) for r in need_upgrade) + max_reg_version_len = max(len(f"v{r['registry_version']}") for r in need_upgrade) + for result in sorted(need_upgrade, key=lambda x: x["package"]): + reg_version = f"v{result['registry_version']}" + pypi_version = f"v{result['pypi_version']}" + print(f" {result['package']:<{max_name_len}} {reg_version:>{max_reg_version_len}} -> {pypi_version}") + if result.get('pr_url'): + print(f" PR: {result['pr_url']}") + else: + print(" (none)") + + if errors: + print(f"\n[X] ERRORS ({len(errors)} packages):") + max_name_len = max(len(r["package"]) for r in errors) + for result in sorted(errors, key=lambda x: x["package"]): + print(f" {result['package']:<{max_name_len}} {result['error']}") + + print("\n" + "=" * 80) + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Check package versions between riscv64 registry and PyPI") + parser.add_argument("packages", nargs="*", help="Specific packages to check (default: all packages from packages.txt)") + parser.add_argument("--create-prs", action="store_true", help="Create pull requests for packages that can be deprecated or upgraded") + parser.add_argument("--summary", action="store_true", help="Show detailed summary at the end") + + args = parser.parse_args() + + if args.packages: + packages = args.packages + print(f"Checking {len(packages)} specified package(s)...") + else: + packages = read_packages() + print("Checking package versions between riscv64 registry and PyPI...") + print(f"Found {len(packages)} packages to check") + + if args.create_prs and not os.environ.get("GH_TOKEN"): + print("[!] Warning: --create-prs specified but GH_TOKEN not set") + args.create_prs = False + + results = [] + for package in packages: + try: + results.append(check_package(package, create_prs=args.create_prs)) + except KeyboardInterrupt: + print("\n[!] Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"[X] Error checking {package}: {e}") + results.append({"status": "error", "package": package, "error": str(e)}) + + if args.summary or len(packages) > 5: + print_summary(results) + + if any(r["status"] == "error" for r in results): + sys.exit(1) + + if args.create_prs: + pr_failures = [ + r for r in results + if r["status"] in ("need_upgrade", "can_deprecate") and r.get("pr_url") is None + ] + if pr_failures: + print(f"\n[X] PR creation failed for {len(pr_failures)} package(s): " + + ", ".join(r["package"] for r in pr_failures)) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci_scripts/deprecated.txt b/ci_scripts/deprecated.txt new file mode 100644 index 0000000..9150faf --- /dev/null +++ b/ci_scripts/deprecated.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2026 BayLibre, SAS +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: Apache-2.0 + +# List of deprecated packages. +# When we deprecated a package, we add it here, to check if upstream is still +# building it + +aiohttp +blake3 +charset-normalizer +cmake +fastar +jiter +kiwisolver +lintrunner +llguidance +lxml +MarkupSafe +maturin +msgpack +nh3 +ninja +optree +patchelf +propcache +pydantic-core +regex +rpds-py +safetensors +scipy-openblas32 +scipy-openblas64 +swig +tokenizers +uv +video_reader-rs +watchfiles +wrapt +xxhash diff --git a/ci_scripts/list-packages.sh b/ci_scripts/list-packages.sh new file mode 100755 index 0000000..2df7b31 --- /dev/null +++ b/ci_scripts/list-packages.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: Apache-2.0 +# +# This script is originally from: +# +# https://gitlab.com/riseproject/python/wheel_builder/-/blob/main/ci_scripts/list-packages.sh +# +# Its purpose is to test installation of each package RISE supports from the +# registry, then list the packages which were actually installed with their +# versions so that any uninstallable packages can be identified. + +set -e + +if [ "$#" -ne 1 ]; then + echo "list-packages name-prefix" + exit 1 +fi + +apt-get update -qq && apt-get install -qq -y python3 python3-venv +python3 -m venv packages-$1-venv +. packages-$1-venv/bin/activate +python -m pip install --upgrade pip +pip install --only-binary=:all: -r packages.txt +pip list > $1-installed.txt diff --git a/ci_scripts/packages.txt b/ci_scripts/packages.txt new file mode 100644 index 0000000..fecac92 --- /dev/null +++ b/ci_scripts/packages.txt @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# SPDX-FileCopyrightText: 2026 The RISE Project + +# SPDX-License-Identifier: Apache-2.0 + +# List of supported packages. +# The name should match the package name on the registry, not the name of the +# project itself. +# The list is parsed by audit.sh script to install packages and audit them. + +argon2-cffi-bindings +cffi +contourpy +cryptography +fastrlock +frozenlist +grpcio +hf-transfer +hf-xet +httptools +libcst +matplotlib +ml-dtypes +msgspec +multidict +numpy +onnx +openai-harmony +orjson +outlines-core +pandas +pillow +polars-runtime-32 +polars-runtime-64 +psutil +PyNaCl +pyuwsgi +PyYAML +pyzmq +rignore +scikit-image +scipy +sentencepiece +setproctitle +soundfile +tiktoken +tlparse +tornado +ujson +uvloop +websockets +z3-solver