From 4bbd9e201518e5c1c787c3109b56dfce012d3076 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 1 Sep 2026 15:32:54 +0100 Subject: [PATCH 1/9] 'gain_directory_name' should be considered a data transfer-related field --- src/murfey/util/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index 564bbe505..30d456f89 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -67,6 +67,7 @@ class MachineConfig(BaseModel): # type: ignore "files": [], } mkdir_chmod: int = 0o2750 + gain_directory_name: str = "processing" # Rsync setup rsync_url: str = "" @@ -85,7 +86,6 @@ class MachineConfig(BaseModel): # type: ignore # General processing setup processing_enabled: bool = True process_by_default: bool = True - gain_directory_name: str = "processing" process_multiple_datasets: bool = True processed_directory_name: str = "processed" processed_extra_directory: str = "" From ec490aae534418305356292ebd2366e2bd9f7615 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 1 Sep 2026 16:51:06 +0100 Subject: [PATCH 2/9] Added API endpoints to the instrument and backend servers to return list of OTF directories found under the configured reference file directory --- src/murfey/instrument_server/api.py | 40 +++++++++++++++++++++++++++++ src/murfey/server/api/instrument.py | 31 ++++++++++++++++++++++ src/murfey/util/route_manifest.yaml | 18 +++++++++++++ 3 files changed, 89 insertions(+) diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index abf92a64c..d6ec8c814 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -383,6 +383,46 @@ def register_processing_parameters( return {"success": True} +@router.get("/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs") +def get_possible_otf_dirs( + instrument_name: str, + session_id: MurfeySessionID, +) -> list[File]: + """ + Looks under the configured reference file directory for all top-level directories, + and returns them as a list of Pydantic models. + """ + # Get the machine config from the backend server + url_path = url_path_for( + "session_control.router", + "machine_info_by_instrument", + instrument_name=sanitise_nonpath(instrument_name), + ) + machine_config = requests.get( + f"{_get_murfey_url()}{url_path}", + headers={"Authorization": f"Bearer {tokens[session_id]}"}, + ).json() + + # Look for OTF directories under the specified directory + candidates: list[File] = [] + for item in secure_path( + Path(machine_config["gain_reference_directory"]), keep_spaces=True + ).glob("*"): + if item.is_dir(): + dir_stats = item.stat() + candidates.append( + File( + name=item.name, + description="", + size=dir_stats.st_size / 1e6, + timestamp=datetime.fromtimestamp(dir_stats.st_mtime), + full_path=str(item), + ) + ) + candidates.sort(key=lambda x: x.timestamp, reverse=True) + return candidates + + @router.get( "/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references" ) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 4b1cd9ac8..7f97423a8 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -377,6 +377,37 @@ async def check_instrument_server(instrument_name: MurfeyInstrumentName): return data +@router.get("/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs") +async def get_possible_otf_dirs( + instrument_name: MurfeyInstrumentName, + session_id: MurfeySessionID, +) -> list[File]: + """ + Submits a GET request to the client-side instrument server to get all top-level + directories found under the configured OTF files directory. + """ + data: list[File] = [] + machine_config = get_machine_config(instrument_name=instrument_name)[ + instrument_name + ] + if machine_config.instrument_server_url: + async with lock: + token = instrument_server_tokens[session_id]["access_token"] + async with aiohttp.ClientSession() as clientsession: + url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name=sanitise(instrument_name), + session_id=session_id, + ) + async with clientsession.get( + f"{machine_config.instrument_server_url}{url_path}", + headers={"Authorization": f"Bearer {token}"}, + ) as resp: + data = await resp.json() + return data + + @router.get( "/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references" ) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index e882fef32..878772a8d 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -130,6 +130,15 @@ murfey.instrument_server.api.router: type: int methods: - POST + - path: /instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs + function: get_possible_otf_dirs + path_params: + - name: instrument_name + type: str + - name: session_id + type: int + methods: + - GET - path: /instruments/{instrument_name}/sessions/{session_id}/possible_gain_references function: get_possible_gain_references path_params: @@ -543,6 +552,15 @@ murfey.server.api.instrument.router: type: str methods: - GET + - path: /instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs + function: get_possible_otf_dirs + path_params: + - name: instrument_name + type: str + - name: session_id + type: int + methods: + - GET - path: /instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references function: get_possible_gain_references path_params: From 0e7c3bc1bad7158ff713073ab0fc11bdac47f899 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 1 Sep 2026 19:01:51 +0100 Subject: [PATCH 3/9] Tidied up existing test code for module --- tests/instrument_server/test_api.py | 36 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index cf48b3ba7..66a0af3e4 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -30,17 +30,17 @@ def set_up_test_client(session_id: Optional[int] = None): return TestClient(client_app) -test_get_murfey_url_params_matrix = ( - # Server URL to use - ("default",), - ("0.0.0.0:8000",), - ("murfey_server",), - ("http://murfey_server:8000",), - ("http://murfey_server:8080/api",), +@pytest.mark.parametrize( + "test_params", + ( + # Server URL to use + ("default",), + ("0.0.0.0:8000",), + ("murfey_server",), + ("http://murfey_server:8000",), + ("http://murfey_server:8080/api",), + ), ) - - -@pytest.mark.parametrize("test_params", test_get_murfey_url_params_matrix) def test_get_murfey_url( test_params: tuple[str], mock_client_configuration, # From conftest.py @@ -101,15 +101,15 @@ def test_check_multigrid_controller_status(mocker: MockerFixture): } -test_upload_gain_reference_params_matrix = ( - # Rsync URL settings - ("http://1.1.1.1",), # When rsync_url is provided - ("",), # When rsync_url is blank - (None,), # When rsync_url not provided +@pytest.mark.parametrize( + "test_params", + ( + # Rsync URL settings + ("http://1.1.1.1",), # When rsync_url is provided + ("",), # When rsync_url is blank + (None,), # When rsync_url not provided + ), ) - - -@pytest.mark.parametrize("test_params", test_upload_gain_reference_params_matrix) def test_upload_gain_reference( mocker: MockerFixture, test_params: tuple[Optional[str]], From 229c8cc2b965a040b827bb2c0e34245ce5725149 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 1 Sep 2026 19:24:51 +0100 Subject: [PATCH 4/9] Added test for the instrument server OTF directory-searching function --- tests/instrument_server/test_api.py | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index 66a0af3e4..47e5ae02a 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from typing import Optional from unittest.mock import ANY, MagicMock, patch @@ -15,6 +16,7 @@ ) from murfey.util import posix_path from murfey.util.api import url_path_for +from murfey.util.config import MachineConfig def set_up_test_client(session_id: Optional[int] = None): @@ -101,6 +103,54 @@ def test_check_multigrid_controller_status(mocker: MockerFixture): } +def test_get_possible_otf_dirs( + mocker: MockerFixture, + tmp_path: Path, +): + session_id = 1 + + # Set up client-side OTF files + otf_dir = tmp_path / "otfs" / "otfs-123456" + otf_dir.mkdir(parents=True, exist_ok=True) + for stem in ("far_red.tiff", "red.tiff", "green.tiff", "blue.tiff"): + file = otf_dir / stem + file.touch(exist_ok=True) + + # Mock the stored tokens + mocker.patch("murfey.instrument_server.api.tokens", {session_id: "dummy"}) + + # Mock the GET request + mock_machine_config = json.loads( + MachineConfig(gain_reference_directory=otf_dir.parent).model_dump_json() + ) + mock_response = MagicMock() + mock_response.json.return_value = mock_machine_config + mocker.patch( + "murfey.instrument_server.api.requests.get", return_value=mock_response + ) + + # Set up the test client + client_server = set_up_test_client(session_id=session_id) + url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name="sim", + session_id=session_id, + ) + response = client_server.get(url_path) + + # Check that the result is as expected + assert response.json() == [ + { + "name": str(otf_dir.name), + "description": "", + "size": ANY, + "timestamp": ANY, + "full_path": str(otf_dir), + } + ] + + @pytest.mark.parametrize( "test_params", ( From c67e7d770c85e99269e433fc9ca76a636c540d51 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 2 Sep 2026 09:38:26 +0100 Subject: [PATCH 5/9] Fixed broken test --- tests/instrument_server/test_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index 47e5ae02a..367118368 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -119,6 +119,12 @@ def test_get_possible_otf_dirs( # Mock the stored tokens mocker.patch("murfey.instrument_server.api.tokens", {session_id: "dummy"}) + # Mock the stored Murfey URL + mocker.patch( + "murfey.instrument_server.api.murfey_server_url", + MagicMock(url="dummy"), + ) + # Mock the GET request mock_machine_config = json.loads( MachineConfig(gain_reference_directory=otf_dir.parent).model_dump_json() From 1dc7496400cb199ca5c2b0c96af529b02851c334 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 2 Sep 2026 09:45:36 +0100 Subject: [PATCH 6/9] Moved test backend server creation logic into a helper function --- tests/server/api/test_instrument.py | 34 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/server/api/test_instrument.py b/tests/server/api/test_instrument.py index a2b022311..696908082 100644 --- a/tests/server/api/test_instrument.py +++ b/tests/server/api/test_instrument.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Callable, Literal from unittest import mock from unittest.mock import AsyncMock, MagicMock @@ -51,6 +51,24 @@ def mock_aiohttp_clientsession( return mock_clientsession, mock_response +def set_up_test_backend_client(session_id: int, mock_db_session: Callable): + """ + Helper function to set up a test backend server whose response can be inspected + to check that the endpoint function works as expected + """ + # Set up the backend server + backend_app = FastAPI() + + # Override validation and database dependencies + backend_app.dependency_overrides[validate_token] = lambda: None + backend_app.dependency_overrides[validate_frontend_session_access] = ( + lambda: session_id + ) + backend_app.dependency_overrides[murfey_db_session] = mock_db_session + backend_app.include_router(backend_router) + return TestClient(backend_app) + + def test_check_multigrid_controller_status(mocker: MockerFixture): # Set up the objects to mock instrument_name = "test" @@ -92,17 +110,11 @@ def mock_get_db_session(): status=200, ) - # Set up the backend server - backend_app = FastAPI() - - # Override validation and database dependencies - backend_app.dependency_overrides[validate_token] = lambda: None - backend_app.dependency_overrides[validate_frontend_session_access] = ( - lambda: session_id + # # Set up the backend server + backend_server = set_up_test_backend_client( + session_id=session_id, + mock_db_session=mock_get_db_session, ) - backend_app.dependency_overrides[murfey_db_session] = mock_get_db_session - backend_app.include_router(backend_router) - backend_server = TestClient(backend_app) # Construct the URL paths for poking and sending to backend_url_path = url_path_for( From f3b42722769c6e4b5b7ff34559cef59f78c5e4c4 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 2 Sep 2026 10:30:51 +0100 Subject: [PATCH 7/9] Added test for backend function to get OTF dirs; updated helper functions to extend mocking logic; fixed typos --- tests/server/api/test_instrument.py | 106 ++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 7 deletions(-) diff --git a/tests/server/api/test_instrument.py b/tests/server/api/test_instrument.py index 696908082..6850768f8 100644 --- a/tests/server/api/test_instrument.py +++ b/tests/server/api/test_instrument.py @@ -6,10 +6,15 @@ from fastapi.testclient import TestClient from pytest_mock import MockerFixture -from murfey.server.api.auth import validate_frontend_session_access, validate_token +from murfey.server.api.auth import ( + validate_frontend_session_access, + validate_token, + validate_user_instrument_access, +) from murfey.server.api.instrument import router as backend_router from murfey.server.murfey_db import murfey_db_session from murfey.util.api import url_path_for +from murfey.util.config import MachineConfig def mock_aiohttp_clientsession( @@ -46,12 +51,17 @@ def mock_aiohttp_clientsession( getattr(mock_clientsession, method.lower()).return_value = mock_context_manager # Patch 'aiohttp.ClientSession' to return the mocked client session - mocker.patch("aiohttp.ClientSession", return_value=mock_clientsession) + mocker.patch( + "murfey.server.api.instrument.aiohttp.ClientSession", + return_value=mock_clientsession, + ) return mock_clientsession, mock_response -def set_up_test_backend_client(session_id: int, mock_db_session: Callable): +def set_up_test_backend_client( + session_id: int, instrument_name: str, mock_db_session: Callable +): """ Helper function to set up a test backend server whose response can be inspected to check that the endpoint function works as expected @@ -61,6 +71,9 @@ def set_up_test_backend_client(session_id: int, mock_db_session: Callable): # Override validation and database dependencies backend_app.dependency_overrides[validate_token] = lambda: None + backend_app.dependency_overrides[validate_user_instrument_access] = ( + lambda: instrument_name + ) backend_app.dependency_overrides[validate_frontend_session_access] = ( lambda: session_id ) @@ -73,7 +86,7 @@ def test_check_multigrid_controller_status(mocker: MockerFixture): # Set up the objects to mock instrument_name = "test" session_id = 1 - instrment_server_url = "https://murfey.instrument-server.test" + instrument_server_url = "https://murfey.instrument-server.test" # Override the database session generator mock_session = MagicMock() @@ -88,7 +101,7 @@ def mock_get_db_session(): # Mock the machine config mock_machine_config = MagicMock() - mock_machine_config.instrument_server_url = instrment_server_url + mock_machine_config.instrument_server_url = instrument_server_url mock_get_machine_config = mocker.patch( "murfey.server.api.instrument.get_machine_config" ) @@ -110,9 +123,10 @@ def mock_get_db_session(): status=200, ) - # # Set up the backend server + # Set up the backend server backend_server = set_up_test_backend_client( session_id=session_id, + instrument_name=instrument_name, mock_db_session=mock_get_db_session, ) @@ -135,8 +149,86 @@ def mock_get_db_session(): mock_db_session.exec.assert_called_once() mock_get_machine_config.assert_called_once_with(instrument_name=instrument_name) mock_clientsession.get.assert_called_once_with( - f"{instrment_server_url}{client_url_path}", + f"{instrument_server_url}{client_url_path}", headers={"Authorization": f"Bearer {mock_tokens[session_id]['access_token']}"}, ) assert response.status_code == 200 assert response.json() == {"exists": True} + + +def test_get_possible_otf_dirs( + mocker: MockerFixture, +): + instrument_name = "sim" + session_id = 1 + instrument_server_url = "https://murfey.instrument-server.test" + access_token = "dummy" + + # Mock the machine config + mock_machine_config = MachineConfig(instrument_server_url=instrument_server_url) + mock_get_machine_config = mocker.patch( + "murfey.server.api.instrument.get_machine_config", + return_value={instrument_name: mock_machine_config}, + ) + + # Mock the instrument server access token + mocker.patch( + "murfey.server.api.instrument.instrument_server_tokens", + {session_id: {"access_token": access_token}}, + ) + + # Mock the client session the API is requesting from + json_data = [ + { + "name": "dummy", + "description": "dummy", + "size": 0, + "timestamp": "2020-01-01T12:34:56", + "full_path": "/path/to/dummy", + } + ] + mock_client_session, _ = mock_aiohttp_clientsession( + mocker, + method="get", + json_data=json_data, + ) + + # Set up the backend server + backend_server = set_up_test_backend_client( + session_id=session_id, + instrument_name=instrument_name, + mock_db_session=lambda: None, + ) + + # Construct the URL paths for poking and sending to + backend_url_path = url_path_for( + "api.instrument.router", + "get_possible_otf_dirs", + instrument_name=instrument_name, + session_id=session_id, + ) + assert ( + backend_url_path + == f"/instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs" + ) + client_url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name=instrument_name, + session_id=session_id, + ) + assert ( + client_url_path + == f"/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs" + ) + + # Poke the backend + response = backend_server.get(backend_url_path) + mock_get_machine_config.assert_called_once() + mock_client_session.get.assert_called_once_with( + f"{instrument_server_url}{client_url_path}", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == 200 + assert response.json() == json_data + pass From 123760150ca64edf1443b189629157093b09ee99 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 2 Sep 2026 11:17:19 +0100 Subject: [PATCH 8/9] Add a check and error log for when gain reference directory is not configured --- src/murfey/instrument_server/api.py | 32 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index d6ec8c814..1b987b8c3 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -398,28 +398,30 @@ def get_possible_otf_dirs( "machine_info_by_instrument", instrument_name=sanitise_nonpath(instrument_name), ) - machine_config = requests.get( + machine_config: dict[str, Any] = requests.get( f"{_get_murfey_url()}{url_path}", headers={"Authorization": f"Bearer {tokens[session_id]}"}, ).json() # Look for OTF directories under the specified directory candidates: list[File] = [] - for item in secure_path( - Path(machine_config["gain_reference_directory"]), keep_spaces=True - ).glob("*"): - if item.is_dir(): - dir_stats = item.stat() - candidates.append( - File( - name=item.name, - description="", - size=dir_stats.st_size / 1e6, - timestamp=datetime.fromtimestamp(dir_stats.st_mtime), - full_path=str(item), + otf_dir = machine_config.get("gain_reference_directory", None) + if otf_dir is not None: + for item in secure_path(Path(otf_dir), keep_spaces=True).glob("*"): + if item.is_dir(): + dir_stats = item.stat() + candidates.append( + File( + name=item.name, + description="", + size=dir_stats.st_size / 1e6, + timestamp=datetime.fromtimestamp(dir_stats.st_mtime), + full_path=str(item), + ) ) - ) - candidates.sort(key=lambda x: x.timestamp, reverse=True) + candidates.sort(key=lambda x: x.timestamp, reverse=True) + else: + logger.error(f"No OTF directory was configured for {instrument_name}") return candidates From 2206bafa4e96ce6a3c34689c405cc30ae18f5325 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 2 Sep 2026 11:32:50 +0100 Subject: [PATCH 9/9] Add test case for when no gain reference directory is configured --- tests/instrument_server/test_api.py | 33 ++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index 367118368..f15d89ef8 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -103,8 +103,13 @@ def test_check_multigrid_controller_status(mocker: MockerFixture): } +@pytest.mark.parametrize( + "has_gain_reference_directory", + (True, False), +) def test_get_possible_otf_dirs( mocker: MockerFixture, + has_gain_reference_directory: bool, tmp_path: Path, ): session_id = 1 @@ -127,7 +132,11 @@ def test_get_possible_otf_dirs( # Mock the GET request mock_machine_config = json.loads( - MachineConfig(gain_reference_directory=otf_dir.parent).model_dump_json() + MachineConfig( + gain_reference_directory=otf_dir.parent + if has_gain_reference_directory + else None + ).model_dump_json() ) mock_response = MagicMock() mock_response.json.return_value = mock_machine_config @@ -146,15 +155,19 @@ def test_get_possible_otf_dirs( response = client_server.get(url_path) # Check that the result is as expected - assert response.json() == [ - { - "name": str(otf_dir.name), - "description": "", - "size": ANY, - "timestamp": ANY, - "full_path": str(otf_dir), - } - ] + assert response.json() == ( + [ + { + "name": str(otf_dir.name), + "description": "", + "size": ANY, + "timestamp": ANY, + "full_path": str(otf_dir), + } + ] + if has_gain_reference_directory + else [] + ) @pytest.mark.parametrize(