diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index abf92a64c..1b987b8c3 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -383,6 +383,48 @@ def register_processing_parameters( return {"success": True} +@router.get("/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs") +def get_possible_otf_dirs( + instrument_name: str, + session_id: MurfeySessionID, +) -> list[File]: + """ + Looks under the configured reference file directory for all top-level directories, + and returns them as a list of Pydantic models. + """ + # Get the machine config from the backend server + url_path = url_path_for( + "session_control.router", + "machine_info_by_instrument", + instrument_name=sanitise_nonpath(instrument_name), + ) + machine_config: dict[str, Any] = requests.get( + f"{_get_murfey_url()}{url_path}", + headers={"Authorization": f"Bearer {tokens[session_id]}"}, + ).json() + + # Look for OTF directories under the specified directory + candidates: list[File] = [] + otf_dir = machine_config.get("gain_reference_directory", None) + if otf_dir is not None: + for item in secure_path(Path(otf_dir), keep_spaces=True).glob("*"): + if item.is_dir(): + dir_stats = item.stat() + candidates.append( + File( + name=item.name, + description="", + size=dir_stats.st_size / 1e6, + timestamp=datetime.fromtimestamp(dir_stats.st_mtime), + full_path=str(item), + ) + ) + candidates.sort(key=lambda x: x.timestamp, reverse=True) + else: + logger.error(f"No OTF directory was configured for {instrument_name}") + return candidates + + @router.get( "/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references" ) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 4b1cd9ac8..7f97423a8 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -377,6 +377,37 @@ async def check_instrument_server(instrument_name: MurfeyInstrumentName): return data +@router.get("/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs") +async def get_possible_otf_dirs( + instrument_name: MurfeyInstrumentName, + session_id: MurfeySessionID, +) -> list[File]: + """ + Submits a GET request to the client-side instrument server to get all top-level + directories found under the configured OTF files directory. + """ + data: list[File] = [] + machine_config = get_machine_config(instrument_name=instrument_name)[ + instrument_name + ] + if machine_config.instrument_server_url: + async with lock: + token = instrument_server_tokens[session_id]["access_token"] + async with aiohttp.ClientSession() as clientsession: + url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name=sanitise(instrument_name), + session_id=session_id, + ) + async with clientsession.get( + f"{machine_config.instrument_server_url}{url_path}", + headers={"Authorization": f"Bearer {token}"}, + ) as resp: + data = await resp.json() + return data + + @router.get( "/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references" ) diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index 564bbe505..30d456f89 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -67,6 +67,7 @@ class MachineConfig(BaseModel): # type: ignore "files": [], } mkdir_chmod: int = 0o2750 + gain_directory_name: str = "processing" # Rsync setup rsync_url: str = "" @@ -85,7 +86,6 @@ class MachineConfig(BaseModel): # type: ignore # General processing setup processing_enabled: bool = True process_by_default: bool = True - gain_directory_name: str = "processing" process_multiple_datasets: bool = True processed_directory_name: str = "processed" processed_extra_directory: str = "" diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index e882fef32..878772a8d 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -130,6 +130,15 @@ murfey.instrument_server.api.router: type: int methods: - POST + - path: /instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs + function: get_possible_otf_dirs + path_params: + - name: instrument_name + type: str + - name: session_id + type: int + methods: + - GET - path: /instruments/{instrument_name}/sessions/{session_id}/possible_gain_references function: get_possible_gain_references path_params: @@ -543,6 +552,15 @@ murfey.server.api.instrument.router: type: str methods: - GET + - path: /instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs + function: get_possible_otf_dirs + path_params: + - name: instrument_name + type: str + - name: session_id + type: int + methods: + - GET - path: /instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_gain_references function: get_possible_gain_references path_params: diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index cf48b3ba7..f15d89ef8 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from typing import Optional from unittest.mock import ANY, MagicMock, patch @@ -15,6 +16,7 @@ ) from murfey.util import posix_path from murfey.util.api import url_path_for +from murfey.util.config import MachineConfig def set_up_test_client(session_id: Optional[int] = None): @@ -30,17 +32,17 @@ def set_up_test_client(session_id: Optional[int] = None): return TestClient(client_app) -test_get_murfey_url_params_matrix = ( - # Server URL to use - ("default",), - ("0.0.0.0:8000",), - ("murfey_server",), - ("http://murfey_server:8000",), - ("http://murfey_server:8080/api",), +@pytest.mark.parametrize( + "test_params", + ( + # Server URL to use + ("default",), + ("0.0.0.0:8000",), + ("murfey_server",), + ("http://murfey_server:8000",), + ("http://murfey_server:8080/api",), + ), ) - - -@pytest.mark.parametrize("test_params", test_get_murfey_url_params_matrix) def test_get_murfey_url( test_params: tuple[str], mock_client_configuration, # From conftest.py @@ -101,15 +103,82 @@ def test_check_multigrid_controller_status(mocker: MockerFixture): } -test_upload_gain_reference_params_matrix = ( - # Rsync URL settings - ("http://1.1.1.1",), # When rsync_url is provided - ("",), # When rsync_url is blank - (None,), # When rsync_url not provided +@pytest.mark.parametrize( + "has_gain_reference_directory", + (True, False), ) +def test_get_possible_otf_dirs( + mocker: MockerFixture, + has_gain_reference_directory: bool, + tmp_path: Path, +): + session_id = 1 + + # Set up client-side OTF files + otf_dir = tmp_path / "otfs" / "otfs-123456" + otf_dir.mkdir(parents=True, exist_ok=True) + for stem in ("far_red.tiff", "red.tiff", "green.tiff", "blue.tiff"): + file = otf_dir / stem + file.touch(exist_ok=True) + + # Mock the stored tokens + mocker.patch("murfey.instrument_server.api.tokens", {session_id: "dummy"}) + + # Mock the stored Murfey URL + mocker.patch( + "murfey.instrument_server.api.murfey_server_url", + MagicMock(url="dummy"), + ) + + # Mock the GET request + mock_machine_config = json.loads( + MachineConfig( + gain_reference_directory=otf_dir.parent + if has_gain_reference_directory + else None + ).model_dump_json() + ) + mock_response = MagicMock() + mock_response.json.return_value = mock_machine_config + mocker.patch( + "murfey.instrument_server.api.requests.get", return_value=mock_response + ) + + # Set up the test client + client_server = set_up_test_client(session_id=session_id) + url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name="sim", + session_id=session_id, + ) + response = client_server.get(url_path) + + # Check that the result is as expected + assert response.json() == ( + [ + { + "name": str(otf_dir.name), + "description": "", + "size": ANY, + "timestamp": ANY, + "full_path": str(otf_dir), + } + ] + if has_gain_reference_directory + else [] + ) -@pytest.mark.parametrize("test_params", test_upload_gain_reference_params_matrix) +@pytest.mark.parametrize( + "test_params", + ( + # Rsync URL settings + ("http://1.1.1.1",), # When rsync_url is provided + ("",), # When rsync_url is blank + (None,), # When rsync_url not provided + ), +) def test_upload_gain_reference( mocker: MockerFixture, test_params: tuple[Optional[str]], diff --git a/tests/server/api/test_instrument.py b/tests/server/api/test_instrument.py index a2b022311..6850768f8 100644 --- a/tests/server/api/test_instrument.py +++ b/tests/server/api/test_instrument.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Callable, Literal from unittest import mock from unittest.mock import AsyncMock, MagicMock @@ -6,10 +6,15 @@ from fastapi.testclient import TestClient from pytest_mock import MockerFixture -from murfey.server.api.auth import validate_frontend_session_access, validate_token +from murfey.server.api.auth import ( + validate_frontend_session_access, + validate_token, + validate_user_instrument_access, +) from murfey.server.api.instrument import router as backend_router from murfey.server.murfey_db import murfey_db_session from murfey.util.api import url_path_for +from murfey.util.config import MachineConfig def mock_aiohttp_clientsession( @@ -46,16 +51,42 @@ def mock_aiohttp_clientsession( getattr(mock_clientsession, method.lower()).return_value = mock_context_manager # Patch 'aiohttp.ClientSession' to return the mocked client session - mocker.patch("aiohttp.ClientSession", return_value=mock_clientsession) + mocker.patch( + "murfey.server.api.instrument.aiohttp.ClientSession", + return_value=mock_clientsession, + ) return mock_clientsession, mock_response +def set_up_test_backend_client( + session_id: int, instrument_name: str, mock_db_session: Callable +): + """ + Helper function to set up a test backend server whose response can be inspected + to check that the endpoint function works as expected + """ + # Set up the backend server + backend_app = FastAPI() + + # Override validation and database dependencies + backend_app.dependency_overrides[validate_token] = lambda: None + backend_app.dependency_overrides[validate_user_instrument_access] = ( + lambda: instrument_name + ) + backend_app.dependency_overrides[validate_frontend_session_access] = ( + lambda: session_id + ) + backend_app.dependency_overrides[murfey_db_session] = mock_db_session + backend_app.include_router(backend_router) + return TestClient(backend_app) + + def test_check_multigrid_controller_status(mocker: MockerFixture): # Set up the objects to mock instrument_name = "test" session_id = 1 - instrment_server_url = "https://murfey.instrument-server.test" + instrument_server_url = "https://murfey.instrument-server.test" # Override the database session generator mock_session = MagicMock() @@ -70,7 +101,7 @@ def mock_get_db_session(): # Mock the machine config mock_machine_config = MagicMock() - mock_machine_config.instrument_server_url = instrment_server_url + mock_machine_config.instrument_server_url = instrument_server_url mock_get_machine_config = mocker.patch( "murfey.server.api.instrument.get_machine_config" ) @@ -93,16 +124,11 @@ def mock_get_db_session(): ) # Set up the backend server - backend_app = FastAPI() - - # Override validation and database dependencies - backend_app.dependency_overrides[validate_token] = lambda: None - backend_app.dependency_overrides[validate_frontend_session_access] = ( - lambda: session_id + backend_server = set_up_test_backend_client( + session_id=session_id, + instrument_name=instrument_name, + mock_db_session=mock_get_db_session, ) - backend_app.dependency_overrides[murfey_db_session] = mock_get_db_session - backend_app.include_router(backend_router) - backend_server = TestClient(backend_app) # Construct the URL paths for poking and sending to backend_url_path = url_path_for( @@ -123,8 +149,86 @@ def mock_get_db_session(): mock_db_session.exec.assert_called_once() mock_get_machine_config.assert_called_once_with(instrument_name=instrument_name) mock_clientsession.get.assert_called_once_with( - f"{instrment_server_url}{client_url_path}", + f"{instrument_server_url}{client_url_path}", headers={"Authorization": f"Bearer {mock_tokens[session_id]['access_token']}"}, ) assert response.status_code == 200 assert response.json() == {"exists": True} + + +def test_get_possible_otf_dirs( + mocker: MockerFixture, +): + instrument_name = "sim" + session_id = 1 + instrument_server_url = "https://murfey.instrument-server.test" + access_token = "dummy" + + # Mock the machine config + mock_machine_config = MachineConfig(instrument_server_url=instrument_server_url) + mock_get_machine_config = mocker.patch( + "murfey.server.api.instrument.get_machine_config", + return_value={instrument_name: mock_machine_config}, + ) + + # Mock the instrument server access token + mocker.patch( + "murfey.server.api.instrument.instrument_server_tokens", + {session_id: {"access_token": access_token}}, + ) + + # Mock the client session the API is requesting from + json_data = [ + { + "name": "dummy", + "description": "dummy", + "size": 0, + "timestamp": "2020-01-01T12:34:56", + "full_path": "/path/to/dummy", + } + ] + mock_client_session, _ = mock_aiohttp_clientsession( + mocker, + method="get", + json_data=json_data, + ) + + # Set up the backend server + backend_server = set_up_test_backend_client( + session_id=session_id, + instrument_name=instrument_name, + mock_db_session=lambda: None, + ) + + # Construct the URL paths for poking and sending to + backend_url_path = url_path_for( + "api.instrument.router", + "get_possible_otf_dirs", + instrument_name=instrument_name, + session_id=session_id, + ) + assert ( + backend_url_path + == f"/instrument_server/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs" + ) + client_url_path = url_path_for( + "api.router", + "get_possible_otf_dirs", + instrument_name=instrument_name, + session_id=session_id, + ) + assert ( + client_url_path + == f"/instruments/{instrument_name}/sessions/{session_id}/possible_otf_dirs" + ) + + # Poke the backend + response = backend_server.get(backend_url_path) + mock_get_machine_config.assert_called_once() + mock_client_session.get.assert_called_once_with( + f"{instrument_server_url}{client_url_path}", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == 200 + assert response.json() == json_data + pass