From 13cfa271b6d871e70c7ed42171dcf8912f3343d0 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sat, 12 Sep 2026 23:27:52 +0100 Subject: [PATCH 1/6] Reproduce embedded Python executable selection on Windows Co-authored-by: Miles Cranmer --- .github/workflows/executable-check.yml | 81 ++++++++++++++++++++++++++ pytest/test_all.py | 34 +++++++++++ 2 files changed, 115 insertions(+) create mode 100644 .github/workflows/executable-check.yml diff --git a/.github/workflows/executable-check.yml b/.github/workflows/executable-check.yml new file mode 100644 index 00000000..7eeeab25 --- /dev/null +++ b/.github/workflows/executable-check.yml @@ -0,0 +1,81 @@ +name: Executable check + +on: + push: + branches: [fix-windows-python-executable] + workflow_dispatch: + +permissions: + contents: read + +jobs: + executable-check: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - name: Windows 3.10 normal + os: windows-latest + python-version: '3.10' + distribution: normal + shell: pwsh + - name: Windows 3.14 normal + os: windows-latest + python-version: '3.14' + distribution: normal + shell: pwsh + - name: Ubuntu 3.10 normal + os: ubuntu-latest + python-version: '3.10' + distribution: normal + shell: bash + - name: Windows 3.10 conda + os: windows-latest + python-version: '3.10' + distribution: conda + shell: pwsh + env: + PYTHON_JULIACALL_HANDLE_SIGNALS: 'yes' + PYTHON_JULIACALL_THREADS: '2' + defaults: + run: + shell: ${{ matrix.shell }} + steps: + - uses: actions/checkout@v7 + + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + + - uses: actions/setup-python@v7 + if: matrix.distribution == 'normal' + with: + python-version: ${{ matrix.python-version }} + + - uses: conda-incubator/setup-miniconda@v4 + if: matrix.distribution == 'conda' + with: + miniforge-variant: Miniforge3 + miniforge-version: latest + activate-environment: executable-check + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + run: | + cp pysrc/juliacall/juliapkg-dev.json pysrc/juliacall/juliapkg.json + python -m pip install --editable . pytest + + - name: Run executable regression + if: runner.os != 'Windows' + run: python -m pytest pytest/test_all.py::test_pythoncall_executable_runs_python -p no:faulthandler + + - name: Run executable regression + if: runner.os == 'Windows' + run: | + python -m pytest pytest/test_all.py::test_pythoncall_executable_runs_python -p no:faulthandler *> pytest-output.txt + $exitCode = $LASTEXITCODE + Get-Content pytest-output.txt + exit $exitCode diff --git a/pytest/test_all.py b/pytest/test_all.py index 9cdc8ce4..751744f4 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -5,6 +5,40 @@ def test_import(): import juliacall +def test_pythoncall_executable_runs_python(): + import os + import subprocess + import sys + + from juliacall import Main as jl + + env = os.environ.copy() + env["JULIA_PYTHONCALL_EXE"] = sys.executable + code = '''\ +using PythonCall +sys = pyimport("sys") +@assert pyconvert(String, sys.executable) == ENV["JULIA_PYTHONCALL_EXE"] +subprocess = pyimport("subprocess") +output = subprocess.check_output([sys.executable, "-c", "print(6, end='')"]) +print(pyconvert(String, output.decode())) +''' + result = subprocess.run( + [ + str(jl.seval("first(Base.julia_cmd().exec)")), + "--project=" + str(jl.seval("dirname(Base.active_project())")), + "--startup-file=no", + "-e", + code, + ], + env=env, + capture_output=True, + text=True, + timeout=180, + ) + assert result.returncode == 0, result.stderr + assert result.stdout == "6" + + def test_newmodule(): import juliacall From a12ac893f46c3ede4469998a9fd8f1d5d23d5768 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sat, 12 Sep 2026 23:32:41 +0100 Subject: [PATCH 2/6] Set the embedded Python executable on Windows 3.10 Co-authored-by: Miles Cranmer --- src/C/context.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/C/context.jl b/src/C/context.jl index 6ccdab89..ea469ffd 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -278,6 +278,16 @@ function init_context() CTX.pyprogname_w = Base.cconvert(Cwstring, CTX.pyprogname) Py_SetProgramName(pointer(CTX.pyprogname_w)) + # Python 3.10 on Windows ignores program_name when resolving the executable. + if Sys.iswindows() && startswith(Base.unsafe_string(Py_GetVersion()), "3.10.") + ccall( + dlsym(CTX.lib_ptr, :_Py_SetProgramFullPath), + Cvoid, + (Ptr{Cwchar_t},), + pointer(CTX.pyprogname_w), + ) + end + # Start the interpreter and register exit hooks Py_InitializeEx(0) atexit() do From 321cb3ac13a5901d658c92ee15427723045421c2 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sat, 12 Sep 2026 23:39:06 +0100 Subject: [PATCH 3/6] Record executable initialization fix and remove diagnostic workflow Co-authored-by: Miles Cranmer --- .github/workflows/executable-check.yml | 81 -------------------------- CHANGELOG.md | 1 + 2 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 .github/workflows/executable-check.yml diff --git a/.github/workflows/executable-check.yml b/.github/workflows/executable-check.yml deleted file mode 100644 index 7eeeab25..00000000 --- a/.github/workflows/executable-check.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Executable check - -on: - push: - branches: [fix-windows-python-executable] - workflow_dispatch: - -permissions: - contents: read - -jobs: - executable-check: - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - include: - - name: Windows 3.10 normal - os: windows-latest - python-version: '3.10' - distribution: normal - shell: pwsh - - name: Windows 3.14 normal - os: windows-latest - python-version: '3.14' - distribution: normal - shell: pwsh - - name: Ubuntu 3.10 normal - os: ubuntu-latest - python-version: '3.10' - distribution: normal - shell: bash - - name: Windows 3.10 conda - os: windows-latest - python-version: '3.10' - distribution: conda - shell: pwsh - env: - PYTHON_JULIACALL_HANDLE_SIGNALS: 'yes' - PYTHON_JULIACALL_THREADS: '2' - defaults: - run: - shell: ${{ matrix.shell }} - steps: - - uses: actions/checkout@v7 - - - uses: julia-actions/setup-julia@v3 - with: - version: '1' - - - uses: actions/setup-python@v7 - if: matrix.distribution == 'normal' - with: - python-version: ${{ matrix.python-version }} - - - uses: conda-incubator/setup-miniconda@v4 - if: matrix.distribution == 'conda' - with: - miniforge-variant: Miniforge3 - miniforge-version: latest - activate-environment: executable-check - python-version: ${{ matrix.python-version }} - - - name: Install test dependencies - run: | - cp pysrc/juliacall/juliapkg-dev.json pysrc/juliacall/juliapkg.json - python -m pip install --editable . pytest - - - name: Run executable regression - if: runner.os != 'Windows' - run: python -m pytest pytest/test_all.py::test_pythoncall_executable_runs_python -p no:faulthandler - - - name: Run executable regression - if: runner.os == 'Windows' - run: | - python -m pytest pytest/test_all.py::test_pythoncall_executable_runs_python -p no:faulthandler *> pytest-output.txt - $exitCode = $LASTEXITCODE - Get-Content pytest-output.txt - exit $exitCode diff --git a/CHANGELOG.md b/CHANGELOG.md index 4051db1b..12d6650a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## Unreleased +* Bug fix: preserve the selected Python executable when embedding Python 3.10 on Windows. * Bug fix: avoid precompilation cache miss when `check_bounds` option set. * Bug fix: `juliacall` set the environment variable `JULIA_PYTHONCALL_EXECUTABLE` instead of `JULIA_PYTHONCALL_EXE`, so child Julia processes resolved a new From 76887cd6f0de280ad6a465de30beab3509162a55 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sun, 13 Sep 2026 01:40:06 +0100 Subject: [PATCH 4/6] Verify PySR worker behavior across interpreter fixes Co-authored-by: Miles Cranmer --- .github/workflows/pysr-worker-check.yml | 94 ++++++++++ pytest/test_all.py | 7 +- scripts/pysr_worker_probe.py | 222 ++++++++++++++++++++++++ 3 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/pysr-worker-check.yml create mode 100644 scripts/pysr_worker_probe.py diff --git a/.github/workflows/pysr-worker-check.yml b/.github/workflows/pysr-worker-check.yml new file mode 100644 index 00000000..ca9176d5 --- /dev/null +++ b/.github/workflows/pysr-worker-check.yml @@ -0,0 +1,94 @@ +name: PySR worker check + +on: + push: + branches: + - fix-windows-python-executable + paths: + - .github/workflows/pysr-worker-check.yml + - scripts/pysr_worker_probe.py + workflow_dispatch: + +jobs: + worker-check: + name: ${{ matrix.variant.mode }} (Python ${{ matrix.python }}) + runs-on: windows-2025 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python: + - "3.10" + - "3.14" + variant: + - mode: legacy0934 + pyjuliacall: "0.9.34" + - mode: pre1362 + pyjuliacall: "0.9.35" + - mode: post1362 + pyjuliacall: "0.9.35" + - mode: upstream-fixed + pyjuliacall: "0.9.35" + + steps: + - uses: actions/checkout@v7 + + - name: Set up Miniforge + uses: conda-incubator/setup-miniconda@v4 + with: + miniforge-variant: Miniforge3 + miniforge-version: latest + activate-environment: pysr-test + python-version: ${{ matrix.python }} + + - name: Install released packages + shell: pwsh + run: conda install --yes --channel conda-forge "python=${{ matrix.python }}" "pysr=2.3.1" "pyjuliacall=${{ matrix.variant.pyjuliacall }}" + + - name: Set up Julia + uses: julia-actions/setup-julia@v3 + with: + version: "1.13.0" + + - name: Install fixed juliacall + if: matrix.variant.mode == 'upstream-fixed' + shell: pwsh + run: python -m pip install --no-deps --editable . + + - name: Set up probe + shell: pwsh + run: | + & cmd /d /c "python scripts/pysr_worker_probe.py --mode ${{ matrix.variant.mode }} --phase setup > worker-setup.log 2>&1" + $exitCode = $LASTEXITCODE + Get-Content -Path worker-setup.log + exit $exitCode + + - name: Run probe + shell: pwsh + run: | + & cmd /d /c "python scripts/pysr_worker_probe.py --mode ${{ matrix.variant.mode }} --phase probe > worker-probe.log 2>&1" + $exitCode = $LASTEXITCODE + Get-Content -Path worker-probe.log + exit $exitCode + + - name: Run full PySR worker tests + if: matrix.variant.mode == 'upstream-fixed' + shell: pwsh + run: | + & cmd /d /c "python -m pysr test main,startup > pysr-tests.log 2>&1" + $exitCode = $LASTEXITCODE + Get-Content -Path pysr-tests.log + exit $exitCode + + - name: Upload diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: pysr-worker-${{ matrix.variant.mode }}-python-${{ matrix.python }} + path: | + worker-setup.log + worker-setup.json + worker-probe.log + worker-observations.json + pysr-tests.log + if-no-files-found: warn diff --git a/pytest/test_all.py b/pytest/test_all.py index 751744f4..151a7361 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -6,18 +6,15 @@ def test_import(): def test_pythoncall_executable_runs_python(): - import os import subprocess import sys from juliacall import Main as jl - env = os.environ.copy() - env["JULIA_PYTHONCALL_EXE"] = sys.executable code = '''\ using PythonCall sys = pyimport("sys") -@assert pyconvert(String, sys.executable) == ENV["JULIA_PYTHONCALL_EXE"] +@assert pyconvert(String, sys.executable) == only(ARGS) subprocess = pyimport("subprocess") output = subprocess.check_output([sys.executable, "-c", "print(6, end='')"]) print(pyconvert(String, output.decode())) @@ -29,8 +26,8 @@ def test_pythoncall_executable_runs_python(): "--startup-file=no", "-e", code, + sys.executable, ], - env=env, capture_output=True, text=True, timeout=180, diff --git a/scripts/pysr_worker_probe.py b/scripts/pysr_worker_probe.py new file mode 100644 index 00000000..d3f073f4 --- /dev/null +++ b/scripts/pysr_worker_probe.py @@ -0,0 +1,222 @@ +"""Temporary Windows comparison of released PySR and the PythonCall checkout.""" + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import traceback + + +MODES = ("legacy0934", "pre1362", "post1362", "upstream-fixed") +MARKER = "WORKER_OBSERVATION=" +COMPATIBILITY_BRANCH = ( + ' if version("juliacall") == "0.9.35":\n' + ' os.environ["JULIA_PYTHONCALL_EXE"] = sys.executable or ""\n' +) +TEST = ( + "pysr.test.test_startup.TestStartup." + "test_juliacall_0935_distributed_worker_uses_current_python" +) +IMPORT_ORDERS = ( + "import pysr\nfrom pysr import jl", + "import juliacall\nimport pysr\nfrom pysr import jl", +) +WORKER_CODE = r''' +using Distributed +worker = only(addprocs(1, exeflags="--threads=1")) +try + fetch(Distributed.remotecall_eval(Main, worker, :(using PythonCall))) + @fetchfrom worker begin + state = pydict( + selected_executable = string(PythonCall.python_executable_path()), + library = string(PythonCall.python_library_path()), + ) + pyexec(""" +import json +import subprocess +import sys +import traceback + +observation = { + "selected_executable": selected_executable, + "library": library, + "executable": sys.executable, + "version": sys.version, + "prefix": sys.prefix, + "sum": sum([1, 2, 3]), +} +try: + result = subprocess.run( + [sys.executable, "-c", "print(6)"], + capture_output=True, text=True, timeout=20, + ) + observation["subprocess"] = { + "outcome": "passed" if result.returncode == 0 else "failed", + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } +except Exception as error: + observation["subprocess"] = { + "outcome": "exception", + "error": repr(error), + "traceback": traceback.format_exc(), + } +result_json = json.dumps(observation) +""", state) + pyconvert(String, state["result_json"]) + end +finally + rmprocs(worker) +end +''' + + +def setup(mode): + spec = importlib.util.find_spec("pysr") + assert spec is not None and spec.submodule_search_locations, "PySR is not installed" + package = Path(next(iter(spec.submodule_search_locations))) + source = package / "julia_import.py" + original = source.read_text(encoding="utf-8") + assert original.count(COMPATIBILITY_BRANCH) == 1, "Expected exactly one PySR 1362 branch" + if mode != "post1362": + source.write_text(original.replace(COMPATIBILITY_BRANCH, ""), encoding="utf-8") + result = { + "mode": mode, + "compatibility_branch_removed": mode != "post1362", + "pysr_source": str(source), + "startup_test_sha256": hashlib.sha256( + (package / "test" / "test_startup.py").read_bytes() + ).hexdigest(), + } + if mode == "upstream-fixed": + import juliacall + import juliapkg + + jl = juliacall.Main + jl.seval("using Pkg") + project = str(jl.seval("Base.active_project()")) + assert Path(project).parent.resolve() == Path(juliapkg.project()).resolve() + assert jl.seval('haskey(Pkg.project().dependencies, "SymbolicRegression")') + jl.Pkg.develop(path=str(Path(__file__).resolve().parents[1])) + assert str(jl.seval("Base.active_project()")) == project + assert jl.seval('haskey(Pkg.project().dependencies, "SymbolicRegression")') + result["julia_project"] = project + result["pythoncall_checkout"] = str(Path(__file__).resolve().parents[1]) + text = json.dumps(result, indent=2) + Path("worker-setup.json").write_text(text + "\n", encoding="utf-8") + print(text, flush=True) + + +def observe(import_order): + import os + + os.environ.pop("JULIA_CONDAPKG_OFFLINE", None) + observation = { + "import_order": import_order, + "parent": { + "executable": sys.executable, + "version": sys.version, + "prefix": sys.prefix, + }, + } + try: + namespace = {} + exec(import_order, namespace) + observation["worker"] = json.loads(namespace["jl"].seval(WORKER_CODE)) + worker = observation["worker"] + observation["checks"] = { + "current_python": worker["executable"] == sys.executable, + "selected_current_python": worker["selected_executable"] == sys.executable, + "current_version": worker["version"] == sys.version, + "current_prefix": worker["prefix"] == sys.prefix, + "sum": worker["sum"] == 6, + "subprocess": ( + worker["subprocess"].get("returncode") == 0 + and worker["subprocess"].get("stdout", "").strip() == "6" + ), + } + observation["outcome"] = ( + "passed" if all(observation["checks"].values()) else "failed" + ) + except Exception as error: + observation.update( + outcome="exception", error=repr(error), traceback=traceback.format_exc() + ) + print(MARKER + json.dumps(observation), flush=True) + return 0 if observation["outcome"] == "passed" else 1 + + +def run_child(arguments): + try: + result = subprocess.run( + [sys.executable, *arguments], capture_output=True, text=True, timeout=300 + ) + return { + "outcome": "passed" if result.returncode == 0 else "failed", + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + except subprocess.TimeoutExpired as error: + return { + "outcome": "timeout", + "returncode": None, + "stdout": (error.stdout or b"").decode(errors="replace"), + "stderr": (error.stderr or b"").decode(errors="replace"), + } + + +def probe(mode): + required_test = mode == "upstream-fixed" or ( + mode == "post1362" and sys.version_info[:2] == (3, 14) + ) + report = { + "mode": mode, + "parent_executable": sys.executable, + "parent_version": sys.version, + "unchanged_test_required": required_test, + "worker_checks_required": mode == "upstream-fixed", + "unchanged_test": run_child(["-m", "unittest", "-v", TEST]), + "import_orders": [], + } + for order in IMPORT_ORDERS: + child = run_child([ + str(Path(__file__).resolve()), "--mode", mode, "--observe", order + ]) + lines = [line for line in child["stdout"].splitlines() if line.startswith(MARKER)] + if len(lines) == 1: + child["observation"] = json.loads(lines[0][len(MARKER):]) + report["import_orders"].append(child) + workers_pass = all( + child["returncode"] == 0 + and child.get("observation", {}).get("outcome") == "passed" + for child in report["import_orders"] + ) + test_pass = report["unchanged_test"]["returncode"] == 0 + report["outcome"] = "passed" if test_pass and workers_pass else "negative" + report["requirements_met"] = ( + (not required_test or test_pass) + and (mode != "upstream-fixed" or workers_pass) + ) + text = json.dumps(report, indent=2) + Path("worker-observations.json").write_text(text + "\n", encoding="utf-8") + print(text, flush=True) + return 0 if report["requirements_met"] else 1 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=MODES, required=True) + parser.add_argument("--phase", choices=("setup", "probe"), default="probe") + parser.add_argument("--observe", choices=IMPORT_ORDERS, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.observe is not None: + sys.exit(observe(args.observe)) + if args.phase == "setup": + setup(args.mode) + else: + sys.exit(probe(args.mode)) From 52c0f58a8839a0c437f68ee1489d9d3fe91bde73 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sun, 13 Sep 2026 01:51:21 +0100 Subject: [PATCH 5/6] Install PySR test dependencies in worker comparison --- .github/workflows/pysr-worker-check.yml | 4 ++++ scripts/pysr_worker_probe.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/pysr-worker-check.yml b/.github/workflows/pysr-worker-check.yml index ca9176d5..34014adb 100644 --- a/.github/workflows/pysr-worker-check.yml +++ b/.github/workflows/pysr-worker-check.yml @@ -45,6 +45,10 @@ jobs: shell: pwsh run: conda install --yes --channel conda-forge "python=${{ matrix.python }}" "pysr=2.3.1" "pyjuliacall=${{ matrix.variant.pyjuliacall }}" + - name: Install test dependencies + shell: pwsh + run: python -m pip install pytest nbval + - name: Set up Julia uses: julia-actions/setup-julia@v3 with: diff --git a/scripts/pysr_worker_probe.py b/scripts/pysr_worker_probe.py index d3f073f4..9280fd93 100644 --- a/scripts/pysr_worker_probe.py +++ b/scripts/pysr_worker_probe.py @@ -114,6 +114,8 @@ def setup(mode): def observe(import_order): import os + os.environ.pop("JULIA_PYTHONCALL_EXE", None) + os.environ.pop("JULIA_PYTHONCALL_EXECUTABLE", None) os.environ.pop("JULIA_CONDAPKG_OFFLINE", None) observation = { "import_order": import_order, @@ -126,6 +128,7 @@ def observe(import_order): try: namespace = {} exec(import_order, namespace) + observation["pythoncall_source"] = str(namespace["jl"].seval("pathof(PythonCall)")) observation["worker"] = json.loads(namespace["jl"].seval(WORKER_CODE)) worker = observation["worker"] observation["checks"] = { From d474dd5155ca9e22211a783ca50de695ac8a19d1 Mon Sep 17 00:00:00 2001 From: MilesCranmerBot Date: Sun, 13 Sep 2026 02:39:03 +0100 Subject: [PATCH 6/6] Remove completed Windows comparison workflow Co-authored-by: Miles Cranmer --- .github/workflows/pysr-worker-check.yml | 98 ----------- scripts/pysr_worker_probe.py | 225 ------------------------ 2 files changed, 323 deletions(-) delete mode 100644 .github/workflows/pysr-worker-check.yml delete mode 100644 scripts/pysr_worker_probe.py diff --git a/.github/workflows/pysr-worker-check.yml b/.github/workflows/pysr-worker-check.yml deleted file mode 100644 index 34014adb..00000000 --- a/.github/workflows/pysr-worker-check.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: PySR worker check - -on: - push: - branches: - - fix-windows-python-executable - paths: - - .github/workflows/pysr-worker-check.yml - - scripts/pysr_worker_probe.py - workflow_dispatch: - -jobs: - worker-check: - name: ${{ matrix.variant.mode }} (Python ${{ matrix.python }}) - runs-on: windows-2025 - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - python: - - "3.10" - - "3.14" - variant: - - mode: legacy0934 - pyjuliacall: "0.9.34" - - mode: pre1362 - pyjuliacall: "0.9.35" - - mode: post1362 - pyjuliacall: "0.9.35" - - mode: upstream-fixed - pyjuliacall: "0.9.35" - - steps: - - uses: actions/checkout@v7 - - - name: Set up Miniforge - uses: conda-incubator/setup-miniconda@v4 - with: - miniforge-variant: Miniforge3 - miniforge-version: latest - activate-environment: pysr-test - python-version: ${{ matrix.python }} - - - name: Install released packages - shell: pwsh - run: conda install --yes --channel conda-forge "python=${{ matrix.python }}" "pysr=2.3.1" "pyjuliacall=${{ matrix.variant.pyjuliacall }}" - - - name: Install test dependencies - shell: pwsh - run: python -m pip install pytest nbval - - - name: Set up Julia - uses: julia-actions/setup-julia@v3 - with: - version: "1.13.0" - - - name: Install fixed juliacall - if: matrix.variant.mode == 'upstream-fixed' - shell: pwsh - run: python -m pip install --no-deps --editable . - - - name: Set up probe - shell: pwsh - run: | - & cmd /d /c "python scripts/pysr_worker_probe.py --mode ${{ matrix.variant.mode }} --phase setup > worker-setup.log 2>&1" - $exitCode = $LASTEXITCODE - Get-Content -Path worker-setup.log - exit $exitCode - - - name: Run probe - shell: pwsh - run: | - & cmd /d /c "python scripts/pysr_worker_probe.py --mode ${{ matrix.variant.mode }} --phase probe > worker-probe.log 2>&1" - $exitCode = $LASTEXITCODE - Get-Content -Path worker-probe.log - exit $exitCode - - - name: Run full PySR worker tests - if: matrix.variant.mode == 'upstream-fixed' - shell: pwsh - run: | - & cmd /d /c "python -m pysr test main,startup > pysr-tests.log 2>&1" - $exitCode = $LASTEXITCODE - Get-Content -Path pysr-tests.log - exit $exitCode - - - name: Upload diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: pysr-worker-${{ matrix.variant.mode }}-python-${{ matrix.python }} - path: | - worker-setup.log - worker-setup.json - worker-probe.log - worker-observations.json - pysr-tests.log - if-no-files-found: warn diff --git a/scripts/pysr_worker_probe.py b/scripts/pysr_worker_probe.py deleted file mode 100644 index 9280fd93..00000000 --- a/scripts/pysr_worker_probe.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Temporary Windows comparison of released PySR and the PythonCall checkout.""" - -import argparse -import hashlib -import importlib.util -import json -from pathlib import Path -import subprocess -import sys -import traceback - - -MODES = ("legacy0934", "pre1362", "post1362", "upstream-fixed") -MARKER = "WORKER_OBSERVATION=" -COMPATIBILITY_BRANCH = ( - ' if version("juliacall") == "0.9.35":\n' - ' os.environ["JULIA_PYTHONCALL_EXE"] = sys.executable or ""\n' -) -TEST = ( - "pysr.test.test_startup.TestStartup." - "test_juliacall_0935_distributed_worker_uses_current_python" -) -IMPORT_ORDERS = ( - "import pysr\nfrom pysr import jl", - "import juliacall\nimport pysr\nfrom pysr import jl", -) -WORKER_CODE = r''' -using Distributed -worker = only(addprocs(1, exeflags="--threads=1")) -try - fetch(Distributed.remotecall_eval(Main, worker, :(using PythonCall))) - @fetchfrom worker begin - state = pydict( - selected_executable = string(PythonCall.python_executable_path()), - library = string(PythonCall.python_library_path()), - ) - pyexec(""" -import json -import subprocess -import sys -import traceback - -observation = { - "selected_executable": selected_executable, - "library": library, - "executable": sys.executable, - "version": sys.version, - "prefix": sys.prefix, - "sum": sum([1, 2, 3]), -} -try: - result = subprocess.run( - [sys.executable, "-c", "print(6)"], - capture_output=True, text=True, timeout=20, - ) - observation["subprocess"] = { - "outcome": "passed" if result.returncode == 0 else "failed", - "returncode": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr, - } -except Exception as error: - observation["subprocess"] = { - "outcome": "exception", - "error": repr(error), - "traceback": traceback.format_exc(), - } -result_json = json.dumps(observation) -""", state) - pyconvert(String, state["result_json"]) - end -finally - rmprocs(worker) -end -''' - - -def setup(mode): - spec = importlib.util.find_spec("pysr") - assert spec is not None and spec.submodule_search_locations, "PySR is not installed" - package = Path(next(iter(spec.submodule_search_locations))) - source = package / "julia_import.py" - original = source.read_text(encoding="utf-8") - assert original.count(COMPATIBILITY_BRANCH) == 1, "Expected exactly one PySR 1362 branch" - if mode != "post1362": - source.write_text(original.replace(COMPATIBILITY_BRANCH, ""), encoding="utf-8") - result = { - "mode": mode, - "compatibility_branch_removed": mode != "post1362", - "pysr_source": str(source), - "startup_test_sha256": hashlib.sha256( - (package / "test" / "test_startup.py").read_bytes() - ).hexdigest(), - } - if mode == "upstream-fixed": - import juliacall - import juliapkg - - jl = juliacall.Main - jl.seval("using Pkg") - project = str(jl.seval("Base.active_project()")) - assert Path(project).parent.resolve() == Path(juliapkg.project()).resolve() - assert jl.seval('haskey(Pkg.project().dependencies, "SymbolicRegression")') - jl.Pkg.develop(path=str(Path(__file__).resolve().parents[1])) - assert str(jl.seval("Base.active_project()")) == project - assert jl.seval('haskey(Pkg.project().dependencies, "SymbolicRegression")') - result["julia_project"] = project - result["pythoncall_checkout"] = str(Path(__file__).resolve().parents[1]) - text = json.dumps(result, indent=2) - Path("worker-setup.json").write_text(text + "\n", encoding="utf-8") - print(text, flush=True) - - -def observe(import_order): - import os - - os.environ.pop("JULIA_PYTHONCALL_EXE", None) - os.environ.pop("JULIA_PYTHONCALL_EXECUTABLE", None) - os.environ.pop("JULIA_CONDAPKG_OFFLINE", None) - observation = { - "import_order": import_order, - "parent": { - "executable": sys.executable, - "version": sys.version, - "prefix": sys.prefix, - }, - } - try: - namespace = {} - exec(import_order, namespace) - observation["pythoncall_source"] = str(namespace["jl"].seval("pathof(PythonCall)")) - observation["worker"] = json.loads(namespace["jl"].seval(WORKER_CODE)) - worker = observation["worker"] - observation["checks"] = { - "current_python": worker["executable"] == sys.executable, - "selected_current_python": worker["selected_executable"] == sys.executable, - "current_version": worker["version"] == sys.version, - "current_prefix": worker["prefix"] == sys.prefix, - "sum": worker["sum"] == 6, - "subprocess": ( - worker["subprocess"].get("returncode") == 0 - and worker["subprocess"].get("stdout", "").strip() == "6" - ), - } - observation["outcome"] = ( - "passed" if all(observation["checks"].values()) else "failed" - ) - except Exception as error: - observation.update( - outcome="exception", error=repr(error), traceback=traceback.format_exc() - ) - print(MARKER + json.dumps(observation), flush=True) - return 0 if observation["outcome"] == "passed" else 1 - - -def run_child(arguments): - try: - result = subprocess.run( - [sys.executable, *arguments], capture_output=True, text=True, timeout=300 - ) - return { - "outcome": "passed" if result.returncode == 0 else "failed", - "returncode": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr, - } - except subprocess.TimeoutExpired as error: - return { - "outcome": "timeout", - "returncode": None, - "stdout": (error.stdout or b"").decode(errors="replace"), - "stderr": (error.stderr or b"").decode(errors="replace"), - } - - -def probe(mode): - required_test = mode == "upstream-fixed" or ( - mode == "post1362" and sys.version_info[:2] == (3, 14) - ) - report = { - "mode": mode, - "parent_executable": sys.executable, - "parent_version": sys.version, - "unchanged_test_required": required_test, - "worker_checks_required": mode == "upstream-fixed", - "unchanged_test": run_child(["-m", "unittest", "-v", TEST]), - "import_orders": [], - } - for order in IMPORT_ORDERS: - child = run_child([ - str(Path(__file__).resolve()), "--mode", mode, "--observe", order - ]) - lines = [line for line in child["stdout"].splitlines() if line.startswith(MARKER)] - if len(lines) == 1: - child["observation"] = json.loads(lines[0][len(MARKER):]) - report["import_orders"].append(child) - workers_pass = all( - child["returncode"] == 0 - and child.get("observation", {}).get("outcome") == "passed" - for child in report["import_orders"] - ) - test_pass = report["unchanged_test"]["returncode"] == 0 - report["outcome"] = "passed" if test_pass and workers_pass else "negative" - report["requirements_met"] = ( - (not required_test or test_pass) - and (mode != "upstream-fixed" or workers_pass) - ) - text = json.dumps(report, indent=2) - Path("worker-observations.json").write_text(text + "\n", encoding="utf-8") - print(text, flush=True) - return 0 if report["requirements_met"] else 1 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--mode", choices=MODES, required=True) - parser.add_argument("--phase", choices=("setup", "probe"), default="probe") - parser.add_argument("--observe", choices=IMPORT_ORDERS, help=argparse.SUPPRESS) - args = parser.parse_args() - if args.observe is not None: - sys.exit(observe(args.observe)) - if args.phase == "setup": - setup(args.mode) - else: - sys.exit(probe(args.mode))