diff --git a/pyproject.toml b/pyproject.toml index 506cf8358..51defe6fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey" AtlasContext = "murfey.client.contexts.atlas:AtlasContext" CLEMContext = "murfey.client.contexts.clem:CLEMContext" FIBContext = "murfey.client.contexts.fib:FIBContext" +SIMContext = "murfey.client.contexts.sim:SIMContext" SPAContext = "murfey.client.contexts.spa:SPAContext" SPAMetadataContext = "murfey.client.contexts.spa_metadata:SPAMetadataContext" SXTContext = "murfey.client.contexts.sxt:SXTContext" diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 9d9182f47..5046fd9dd 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -215,6 +215,38 @@ def _find_context(self, file_path: Path) -> bool: ) return True + # ----------------------------------------------------------------------------- + # SIM workflow checks + # ----------------------------------------------------------------------------- + if ( + # CryoSIM raw data files have no extension, and end with specific suffixes + not file_path.suffix + and file_path.stem.endswith( + ( + # Bright field + "_BF", + # Fluorescent + "_BR", + "_BFR", + "_GR", + "_GFR", + "_BR_FL", + "_BFR_FL", + "_GR_FL", + "_GFR_FL", + ) + ) + ): + if (context := _get_context("SIMContext")) is None: + return False + self._context = context.load()( + "sim", + self._basepath, + self._murfey_config, + self._token, + ) + return True + # ----------------------------------------------------------------------------- # SXT workflow checks # ----------------------------------------------------------------------------- diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index 699b9451e..81f6bd83f 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -1,8 +1,9 @@ import logging from pathlib import Path -from murfey.client.context import Context +from murfey.client.context import Context, _file_transferred_to, _get_source from murfey.client.instance_environment import MurfeyInstanceEnvironment +from murfey.util.client import capture_post logger = logging.getLogger("murfey.client.contexts.sim") @@ -25,4 +26,51 @@ def post_transfer( environment: MurfeyInstanceEnvironment | None = None, **kwargs, ): + super().post_transfer(transferred_file, environment=environment, **kwargs) + if environment is None: + logger.warning("No environment passed in") + return None + + # Look for raw data files + # These have no extensions, and end with one of the listed suffixes + if not transferred_file.suffix and transferred_file.stem.endswith( + ( + # Fluorescent SIM raw data files end as follows + "_BR", + "_BFR", + "_GR", + "_GFR", + "_BR_FL", + "_BFR_FL", + "_GR_FL", + "_GFR_FL", + ) + ): + source = _get_source(transferred_file, environment) + if source is None: + logger.warning(f"No source found for file {transferred_file}") + return None + destination_file = _file_transferred_to( + environment=environment, + source=source, + file_path=transferred_file, + rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")), + ) + + # Submit fluorescent raw data files for processing + logger.info(f"Requesting processing for {transferred_file.name!r}") + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow_sim.router", + function_name="request_sim_processing", + token=self._token, + instrument_name=environment.instrument_name, + data={ + "file": f"{destination_file}", + }, + # Endpoint kwargs + session_id=environment.murfey_session, + ) + return None + return None diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py new file mode 100644 index 000000000..8bc91bc07 --- /dev/null +++ b/src/murfey/server/api/workflow_sim.py @@ -0,0 +1,51 @@ +import json +import logging +from pathlib import Path + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from murfey.server import _transport_object +from murfey.server.api.auth import validate_instrument_token +from murfey.util import sanitise_path + +logger = logging.getLogger("murfey.server.api.workflow_sim") + +router = APIRouter( + prefix="/workflow/sim", + dependencies=[Depends(validate_instrument_token)], + tags=["Workflows: CryoSIM"], +) + + +class SIMDataFile(BaseModel): + file: Path + + +@router.post("/sessions/{session_id}/process_data") +def request_sim_processing(session_id: int, sim_data: SIMDataFile): + if _transport_object is None: + logger.error("No TransportManager object was set up") + return None + + # Construct message and submit it to 'processing_recipe' + logger.info( + f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}" + ) + recipe = { + "recipes": ["sim-process-data"], + "parameters": { + # Job parameters + "session_id": session_id, + "file": f"{str(sim_data.file)}", + "feedback_queue": _transport_object.feedback_queue, + }, + } + logger.debug( + "Will submit the following message to 'processing_recipe':\n" + f"{json.dumps(recipe, indent=2, default=str)}" + ) + # Disabled for now; will submit message once recipe and service have been set up + # _transport_object.send( + # queue="processing_recipe", message=recipe, new_connection=True + # ) diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index c14be9f2c..0313a481b 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -27,6 +27,7 @@ import murfey.server.api.workflow import murfey.server.api.workflow_clem import murfey.server.api.workflow_fib +import murfey.server.api.workflow_sim import murfey.server.api.workflow_sxt from murfey.server import template_files from murfey.util.config import get_security_config @@ -100,6 +101,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.workflow.tomo_router) app.include_router(murfey.server.api.workflow_clem.router) app.include_router(murfey.server.api.workflow_fib.router) +app.include_router(murfey.server.api.workflow_sim.router) app.include_router(murfey.server.api.workflow_sxt.router) app.include_router(murfey.server.api.prometheus.router) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index e80e1237b..bb8641d9e 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1445,6 +1445,14 @@ murfey.server.api.workflow_fib.router: type: int methods: - POST +murfey.server.api.workflow_sim.router: + - path: /workflow/sim/sessions/{session_id}/process_data + function: request_sim_processing + path_params: + - name: session_id + type: int + methods: + - POST murfey.server.api.workflow_sxt.router: - path: /workflow/sxt/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series function: process_sxt_tilt_series diff --git a/tests/client/contexts/test_sim.py b/tests/client/contexts/test_sim.py index d0e8beb38..80acf2d93 100644 --- a/tests/client/contexts/test_sim.py +++ b/tests/client/contexts/test_sim.py @@ -1,7 +1,83 @@ from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock +import pytest +from pytest_mock import MockerFixture + +from murfey.client.context import _file_transferred_to, _get_source from murfey.client.contexts.sim import SIMContext +visit_name = "cm12345-6" +instrument_name = "sim" +session_id = 1 + + +@pytest.fixture +def visit_dir(tmp_path: Path): + return tmp_path / visit_name + + +@pytest.fixture +def sim_data(visit_dir: Path): + file_list = [] + for path in [ + # Bright field + "raw/CtrlApr_G2/20260703_132856_CtrlApr_G2_F3A_BF", + # Fluorescent + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", + "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", + "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL", + "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL", + "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR_FL", + ]: + file = visit_dir / path + file.parent.mkdir(parents=True, exist_ok=True) + file.touch(exist_ok=True) + file_list.append(file) + return file_list + + +def test_get_source( + tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], +): + # Mock the MurfeyInstanceEnvironment + mock_environment = MagicMock() + mock_environment.sources = [ + visit_dir, + tmp_path / "another_dir", + ] + # Check that the correct source directory is found + for file in sim_data: + assert _get_source(file, mock_environment) == visit_dir + + +def test_file_transferred_to( + tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], +): + # Mock the environment + mock_environment = MagicMock() + mock_environment.default_destinations = {visit_dir: "current_year"} + mock_environment.visit = visit_name + + # Iterate across the FIB files to compare against + destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name + for file in sim_data: + # Work out what the expected destination will be + assert _file_transferred_to( + environment=mock_environment, + source=visit_dir, + file_path=file, + rsync_basepath=tmp_path / "sim" / "data", + ) == destination_dir / file.relative_to(visit_dir) + def test_sim_context_initialises(tmp_path: Path): # Initialise the context with dummy variables @@ -20,33 +96,92 @@ def test_sim_context_initialises(tmp_path: Path): assert context.name == "SIMContext" +@pytest.mark.parametrize( + "test_params", + ( # Has environment | Has source + # Success case + (True, True), + # Fail cases + (True, False), # No source + (False, True), # No environment + ), +) def test_post_transfer( + mocker: MockerFixture, + test_params: tuple[bool, bool], tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], ): - """ - NOTE: This is just a basic test for coverage purposes, and will be rewritten - as the SIMContext logic evolves and matures. - """ - # Create a dummy file - base_path = tmp_path - visit_dir = base_path / "visit" - test_file = visit_dir / "raw" / "dummy.txt" - test_file.parent.mkdir(parents=True, exist_ok=True) - test_file.touch(exist_ok=True) + # Unpack test params + use_env, has_src = test_params - # Create other mock variables - machine_config = {"dummy": "dummy"} + # Mock the environment + mock_environment = None + if use_env: + mock_environment = MagicMock() + mock_environment.visit = visit_name + mock_environment.instrument_name = instrument_name + mock_environment.murfey_session = session_id + + # Mock the logger to check if specific logs are triggered + mock_logger = mocker.patch("murfey.client.contexts.sim.logger") + # Iterate across the FIB files to compare against + destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name + destination_files = [ + destination_dir / file.relative_to(visit_dir) + for file in sim_data + if not file.stem.endswith("_BF") + ] + + # Mock the functions used in 'post_transfer' + mock_get_source = mocker.patch("murfey.client.contexts.sim._get_source") + mock_get_source.return_value = tmp_path if has_src else None + + mock_file_transferred_to = mocker.patch( + "murfey.client.contexts.sim._file_transferred_to", side_effect=destination_files + ) + + mock_capture_post = mocker.patch("murfey.client.contexts.sim.capture_post") + + # Initialise the SIMContext + basepath = tmp_path context = SIMContext( - "sim", - basepath=base_path, - machine_config=machine_config, + acquisition_software="sim", + basepath=basepath, + machine_config={}, token="dummy", ) - assert ( - context.post_transfer( - transferred_file=test_file, - environment=None, - ) - is None - ) + # Pass the list of files through + for file in sim_data: + context.post_transfer(file, environment=mock_environment) + if not use_env: + mock_logger.warning.assert_called_with("No environment passed in") + elif not has_src: + mock_logger.warning.assert_called_with(f"No source found for file {file}") + else: + for src, dst in zip(sim_data, [Path(""), *destination_files]): + if src.stem.endswith("_BF"): + continue + else: + mock_get_source.assert_any_call(src, mock_environment) + mock_file_transferred_to.assert_any_call( + environment=mock_environment, + source=basepath, + file_path=src, + rsync_basepath=Path(""), + ) + mock_capture_post.assert_any_call( + base_url=mock.ANY, + router_name="workflow_sim.router", + function_name="request_sim_processing", + token=context._token, + instrument_name=instrument_name, + data={ + "file": f"{dst}", + }, + # Endpoint kwargs + session_id=session_id, + ) + assert mock_capture_post.call_count == len(sim_data) - 1 diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 5deda832b..0694c0569 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -36,6 +36,19 @@ "visit/maps/visit/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", "visit/maps/visit/LayersData/Layer/Electron Snapshot (2)/Electron Snapshot (2).tiff", ], + "SIMContext": [ + # Bright field + "visit/raw/CtrlApr_G2/20260703_132856_CtrlApr_G2_F3A_BF", + # Fluorescent + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", + "visit/raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", + "visit/raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL", + "visit/raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL", + "visit/raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR_FL", + ], "SXTContext": [ "visit/tomo__tag_ROI10_area1_angle-60to60@1.5_1sec_251p.txrm", "visit/X-ray_mosaic_ROI2.xrm", @@ -263,6 +276,35 @@ def test_analyse_fib( assert mock_post_transfer.call_count == len(test_files) +def test_analyse_sim( + mocker: MockerFixture, + tmp_path: Path, +): + # Load the example files corresponding to the SIM workflow + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == "SIMContext" + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert spy_find_context.call_count == 1 + assert analyser._context is not None and "SIMContext" in str(analyser._context) + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) + + def test_analyse_sxt( mocker: MockerFixture, tmp_path: Path, diff --git a/tests/server/api/test_workflow_sim.py b/tests/server/api/test_workflow_sim.py new file mode 100644 index 000000000..a17d7fd92 --- /dev/null +++ b/tests/server/api/test_workflow_sim.py @@ -0,0 +1,60 @@ +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from pytest_mock import MockerFixture + +from murfey.server.api.workflow_sim import SIMDataFile, request_sim_processing + + +@pytest.mark.parametrize("has_transport_object", (True, False)) +def test_request_sim_processing( + mocker: MockerFixture, tmp_path: Path, has_transport_object: bool +): + # Set up the variables + session_id = 1 + sim_data = SIMDataFile(**{"file": str(tmp_path / "dummy")}) + + # Mock the logger + mock_logger = mocker.patch("murfey.server.api.workflow_sim.logger") + + # Mock the transport object + if has_transport_object: + mock_transport_object = MagicMock() + mock_transport_object.feedback_queue = "dummy" + mocker.patch( + "murfey.server.api.workflow_sim._transport_object", + mock_transport_object, + ) + else: + mocker.patch( + "murfey.server.api.workflow_sim._transport_object", + None, + ) + + # Run the function and check that the expected calls were made + request_sim_processing( + session_id=session_id, + sim_data=sim_data, + ) + + # Check that the expected calls were made + if has_transport_object: + recipe = { + "recipes": ["sim-process-data"], + "parameters": { + "session_id": session_id, + "file": f"{sim_data.file}", + "feedback_queue": "dummy", + }, + } + mock_logger.debug.assert_called_with( + "Will submit the following message to 'processing_recipe':\n" + f"{json.dumps(recipe, indent=2, default=str)}" + ) + # mock_transport_object.send.assert_called_with( + # queue="processing_recipe", message=recipe, new_connection=True + # ) + else: + mock_logger.error.assert_called_with("No TransportManager object was set up")