Trigger cryoSIM processing

pyproject.toml

@@ -101,6 +101,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey"
 AtlasContext = "murfey.client.contexts.atlas:AtlasContext"
 CLEMContext = "murfey.client.contexts.clem:CLEMContext"
 FIBContext = "murfey.client.contexts.fib:FIBContext"
+SIMContext = "murfey.client.contexts.sim:SIMContext"
 SPAContext = "murfey.client.contexts.spa:SPAContext"
 SPAMetadataContext = "murfey.client.contexts.spa_metadata:SPAMetadataContext"
 SXTContext = "murfey.client.contexts.sxt:SXTContext"

src/murfey/client/analyser.py

@@ -215,6 +215,38 @@ def _find_context(self, file_path: Path) -> bool:
             )
             return True
 
+        # -----------------------------------------------------------------------------
+        # SIM workflow checks
+        # -----------------------------------------------------------------------------
+        if (
+            # CryoSIM raw data files have no extension, and end with specific suffixes
+            not file_path.suffix
+            and file_path.stem.endswith(
+                (
+                    # Bright field
+                    "_BF",
+                    # Fluorescent
+                    "_BR",
+                    "_BFR",
+                    "_GR",
+                    "_GFR",
+                    "_BR_FL",
+                    "_BFR_FL",
+                    "_GR_FL",
+                    "_GFR_FL",
+                )
+            )
+        ):
+            if (context := _get_context("SIMContext")) is None:
+                return False
+            self._context = context.load()(
+                "sim",
+                self._basepath,
+                self._murfey_config,
+                self._token,
+            )
+            return True
+
         # -----------------------------------------------------------------------------
         # SXT workflow checks
         # -----------------------------------------------------------------------------

src/murfey/client/contexts/sim.py

@@ -1,8 +1,9 @@
 import logging
 from pathlib import Path
 
-from murfey.client.context import Context
+from murfey.client.context import Context, _file_transferred_to, _get_source
 from murfey.client.instance_environment import MurfeyInstanceEnvironment
+from murfey.util.client import capture_post
 
 logger = logging.getLogger("murfey.client.contexts.sim")
 
@@ -25,4 +26,51 @@ def post_transfer(
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ):
+        super().post_transfer(transferred_file, environment=environment, **kwargs)
+        if environment is None:
+            logger.warning("No environment passed in")
+            return None
+
+        # Look for raw data files
+        # These have no extensions, and end with one of the listed suffixes
+        if not transferred_file.suffix and transferred_file.stem.endswith(
+            (
+                # Fluorescent SIM raw data files end as follows
+                "_BR",
+                "_BFR",
+                "_GR",
+                "_GFR",
+                "_BR_FL",
+                "_BFR_FL",
+                "_GR_FL",
+                "_GFR_FL",
+            )
+        ):
+            source = _get_source(transferred_file, environment)
+            if source is None:
+                logger.warning(f"No source found for file {transferred_file}")
+                return None
+            destination_file = _file_transferred_to(
+                environment=environment,
+                source=source,
+                file_path=transferred_file,
+                rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")),
+            )
+
+            # Submit fluorescent raw data files for processing
+            logger.info(f"Requesting processing for {transferred_file.name!r}")
+            capture_post(
+                base_url=str(environment.url.geturl()),
+                router_name="workflow_sim.router",
+                function_name="request_sim_processing",
+                token=self._token,
+                instrument_name=environment.instrument_name,
+                data={
+                    "file": f"{destination_file}",
+                },
+                # Endpoint kwargs
+                session_id=environment.murfey_session,
+            )
+            return None
+
         return None

src/murfey/server/api/workflow_sim.py

@@ -0,0 +1,51 @@
+import json
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+
+from murfey.server import _transport_object
+from murfey.server.api.auth import validate_instrument_token
+from murfey.util import sanitise_path
+
+logger = logging.getLogger("murfey.server.api.workflow_sim")
+
+router = APIRouter(
+    prefix="/workflow/sim",
+    dependencies=[Depends(validate_instrument_token)],
+    tags=["Workflows: CryoSIM"],
+)
+
+
+class SIMDataFile(BaseModel):
+    file: Path
+
+
+@router.post("/sessions/{session_id}/process_data")
+def request_sim_processing(session_id: int, sim_data: SIMDataFile):
+    if _transport_object is None:
+        logger.error("No TransportManager object was set up")
+        return None
+
+    # Construct message and submit it to 'processing_recipe'
+    logger.info(
+        f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}"
+    )
+    recipe = {
+        "recipes": ["sim-process-data"],
+        "parameters": {
+            # Job parameters
+            "session_id": session_id,
+            "file": f"{str(sim_data.file)}",
+            "feedback_queue": _transport_object.feedback_queue,
+        },
+    }
+    logger.debug(
+        "Will submit the following message to 'processing_recipe':\n"
+        f"{json.dumps(recipe, indent=2, default=str)}"
+    )
+    # Disabled for now; will submit message once recipe and service have been set up
+    # _transport_object.send(
+    #     queue="processing_recipe", message=recipe, new_connection=True
+    # )

src/murfey/server/main.py

@@ -27,6 +27,7 @@
 import murfey.server.api.workflow
 import murfey.server.api.workflow_clem
 import murfey.server.api.workflow_fib
+import murfey.server.api.workflow_sim
 import murfey.server.api.workflow_sxt
 from murfey.server import template_files
 from murfey.util.config import get_security_config
@@ -100,6 +101,7 @@ class Settings(BaseSettings):
 app.include_router(murfey.server.api.workflow.tomo_router)
 app.include_router(murfey.server.api.workflow_clem.router)
 app.include_router(murfey.server.api.workflow_fib.router)
+app.include_router(murfey.server.api.workflow_sim.router)
 app.include_router(murfey.server.api.workflow_sxt.router)
 
 app.include_router(murfey.server.api.prometheus.router)

src/murfey/util/route_manifest.yaml

@@ -1445,6 +1445,14 @@ murfey.server.api.workflow_fib.router:
         type: int
     methods:
       - POST
+murfey.server.api.workflow_sim.router:
+  - path: /workflow/sim/sessions/{session_id}/process_data
+    function: request_sim_processing
+    path_params:
+      - name: session_id
+        type: int
+    methods:
+      - POST
 murfey.server.api.workflow_sxt.router:
   - path: /workflow/sxt/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series
     function: process_sxt_tilt_series

tests/client/contexts/test_sim.py

@@ -1,7 +1,83 @@
 from pathlib import Path
+from unittest import mock
+from unittest.mock import MagicMock
 
+import pytest
+from pytest_mock import MockerFixture
+
+from murfey.client.context import _file_transferred_to, _get_source
 from murfey.client.contexts.sim import SIMContext
 
+visit_name = "cm12345-6"
+instrument_name = "sim"
+session_id = 1
+
+
+@pytest.fixture
+def visit_dir(tmp_path: Path):
+    return tmp_path / visit_name
+
+
+@pytest.fixture
+def sim_data(visit_dir: Path):
+    file_list = []
+    for path in [
+        # Bright field
+        "raw/CtrlApr_G2/20260703_132856_CtrlApr_G2_F3A_BF",
+        # Fluorescent
+        "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR",
+        "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR",
+        "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR",
+        "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR",
+        "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL",
+        "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL",
+        "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL",
+        "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR_FL",
+    ]:
+        file = visit_dir / path
+        file.parent.mkdir(parents=True, exist_ok=True)
+        file.touch(exist_ok=True)
+        file_list.append(file)
+    return file_list
+
+
+def test_get_source(
+    tmp_path: Path,
+    visit_dir: Path,
+    sim_data: list[Path],
+):
+    # Mock the MurfeyInstanceEnvironment
+    mock_environment = MagicMock()
+    mock_environment.sources = [
+        visit_dir,
+        tmp_path / "another_dir",
+    ]
+    # Check that the correct source directory is found
+    for file in sim_data:
+        assert _get_source(file, mock_environment) == visit_dir
+
+
+def test_file_transferred_to(
+    tmp_path: Path,
+    visit_dir: Path,
+    sim_data: list[Path],
+):
+    # Mock the environment
+    mock_environment = MagicMock()
+    mock_environment.default_destinations = {visit_dir: "current_year"}
+    mock_environment.visit = visit_name
+
+    # Iterate across the FIB files to compare against
+    destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name
+    for file in sim_data:
+        # Work out what the expected destination will be
+        assert _file_transferred_to(
+            environment=mock_environment,
+            source=visit_dir,
+            file_path=file,
+            rsync_basepath=tmp_path / "sim" / "data",
+        ) == destination_dir / file.relative_to(visit_dir)
+
 
 def test_sim_context_initialises(tmp_path: Path):
     # Initialise the context with dummy variables
@@ -20,33 +96,92 @@ def test_sim_context_initialises(tmp_path: Path):
     assert context.name == "SIMContext"
 
 
+@pytest.mark.parametrize(
+    "test_params",
+    (  # Has environment | Has source
+        # Success case
+        (True, True),
+        # Fail cases
+        (True, False),  # No source
+        (False, True),  # No environment
+    ),
+)
 def test_post_transfer(
+    mocker: MockerFixture,
+    test_params: tuple[bool, bool],
     tmp_path: Path,
+    visit_dir: Path,
+    sim_data: list[Path],
 ):
-    """
-    NOTE: This is just a basic test for coverage purposes, and will be rewritten
-    as the SIMContext logic evolves and matures.
-    """
-    # Create a dummy file
-    base_path = tmp_path
-    visit_dir = base_path / "visit"
-    test_file = visit_dir / "raw" / "dummy.txt"
-    test_file.parent.mkdir(parents=True, exist_ok=True)
-    test_file.touch(exist_ok=True)
+    # Unpack test params
+    use_env, has_src = test_params
 
-    # Create other mock variables
-    machine_config = {"dummy": "dummy"}
+    # Mock the environment
+    mock_environment = None
+    if use_env:
+        mock_environment = MagicMock()
+        mock_environment.visit = visit_name
+        mock_environment.instrument_name = instrument_name
+        mock_environment.murfey_session = session_id
+
+    # Mock the logger to check if specific logs are triggered
+    mock_logger = mocker.patch("murfey.client.contexts.sim.logger")
 
+    # Iterate across the FIB files to compare against
+    destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name
+    destination_files = [
+        destination_dir / file.relative_to(visit_dir)
+        for file in sim_data
+        if not file.stem.endswith("_BF")
+    ]
+
+    # Mock the functions used in 'post_transfer'
+    mock_get_source = mocker.patch("murfey.client.contexts.sim._get_source")
+    mock_get_source.return_value = tmp_path if has_src else None
+
+    mock_file_transferred_to = mocker.patch(
+        "murfey.client.contexts.sim._file_transferred_to", side_effect=destination_files
+    )
+
+    mock_capture_post = mocker.patch("murfey.client.contexts.sim.capture_post")
+
+    # Initialise the SIMContext
+    basepath = tmp_path
     context = SIMContext(
-        "sim",
-        basepath=base_path,
-        machine_config=machine_config,
+        acquisition_software="sim",
+        basepath=basepath,
+        machine_config={},
         token="dummy",
     )
-    assert (
-        context.post_transfer(
-            transferred_file=test_file,
-            environment=None,
-        )
-        is None
-    )
+    # Pass the list of files through
+    for file in sim_data:
+        context.post_transfer(file, environment=mock_environment)
+    if not use_env:
+        mock_logger.warning.assert_called_with("No environment passed in")
+    elif not has_src:
+        mock_logger.warning.assert_called_with(f"No source found for file {file}")
+    else:
+        for src, dst in zip(sim_data, [Path(""), *destination_files]):
+            if src.stem.endswith("_BF"):
+                continue
+            else:
+                mock_get_source.assert_any_call(src, mock_environment)
+                mock_file_transferred_to.assert_any_call(
+                    environment=mock_environment,
+                    source=basepath,
+                    file_path=src,
+                    rsync_basepath=Path(""),
+                )
+                mock_capture_post.assert_any_call(
+                    base_url=mock.ANY,
+                    router_name="workflow_sim.router",
+                    function_name="request_sim_processing",
+                    token=context._token,
+                    instrument_name=instrument_name,
+                    data={
+                        "file": f"{dst}",
+                    },
+                    # Endpoint kwargs
+                    session_id=session_id,
+                )
+        assert mock_capture_post.call_count == len(sim_data) - 1