From c004aa2feb56ed9b9ae4d88bb296d9e31593efd2 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 30 Apr 2026 15:39:35 +0100 Subject: [PATCH 01/14] Add sxt router and rename recipe --- src/murfey/client/contexts/sxt.py | 2 +- src/murfey/server/main.py | 1 + src/murfey/workflows/sxt/process_sxt_tilt_series.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 959643501..60edf5228 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -85,7 +85,7 @@ def register_sxt_data_collection( ) recipes_to_assign_pjids = [ - "sxt-tomo-align", + "sxt-aretomo", ] for recipe in recipes_to_assign_pjids: capture_post( diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index f91109e15..b90dea780 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -96,6 +96,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.workflow.router) app.include_router(murfey.server.api.workflow.correlative_router) app.include_router(murfey.server.api.workflow.spa_router) +app.include_router(murfey.server.api.workflow.sxt_router) app.include_router(murfey.server.api.workflow.tomo_router) app.include_router(murfey.server.api.clem.router) app.include_router(murfey.server.api.workflow_fib.router) diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index 53cf7e9c4..04674822c 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -64,7 +64,7 @@ def process_sxt_tilt_series_workflow( .where(DataCollection.dcg_id == DataCollectionGroup.id) .where(ProcessingJob.dc_id == DataCollection.id) .where(AutoProcProgram.pj_id == ProcessingJob.id) - .where(ProcessingJob.recipe == "sxt-tomo-align") + .where(ProcessingJob.recipe == "sxt-aretomo") ).one() instrument_name = ( murfey_db.exec(select(Session).where(Session.id == session_id)) @@ -93,7 +93,7 @@ def process_sxt_tilt_series_workflow( ) stack_file.parent.mkdir(parents=True, exist_ok=True) zocalo_message = { - "recipes": ["sxt-tomo-align"], + "recipes": ["sxt-aretomo"], "parameters": { "txrm_file": tilt_series_info.txrm, "dcid": collected_ids[1].id, From cd5a03b8e96161329ea26c37194276a8aa05d221 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 30 Apr 2026 15:46:10 +0100 Subject: [PATCH 02/14] Update tests --- tests/client/contexts/test_sxt.py | 2 +- tests/workflows/sxt/test_process_sxt_tilt_series.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py index c50e18abf..c1de8f1ec 100644 --- a/tests/client/contexts/test_sxt.py +++ b/tests/client/contexts/test_sxt.py @@ -107,7 +107,7 @@ def test_sxt_context_txrm( json={ "tag": "example", "source": str(tmp_path), - "recipe": "sxt-tomo-align", + "recipe": "sxt-aretomo", "experiment_type": "sxt", }, headers={"Authorization": "Bearer "}, diff --git a/tests/workflows/sxt/test_process_sxt_tilt_series.py b/tests/workflows/sxt/test_process_sxt_tilt_series.py index c14a62200..f45c4be0d 100644 --- a/tests/workflows/sxt/test_process_sxt_tilt_series.py +++ b/tests/workflows/sxt/test_process_sxt_tilt_series.py @@ -38,7 +38,7 @@ def set_up_db(murfey_db_session: Session): ProcessingJob, lookup_kwargs={ "id": 1, - "recipe": "sxt-tomo-align", + "recipe": "sxt-aretomo", "dc_id": dc_entry.id, }, ) @@ -92,7 +92,7 @@ def test_process_new_sxt_tilt_series( "manual_tilt_offset": -1, "node_creator_queue": "node_creator", }, - "recipes": ["sxt-tomo-align"], + "recipes": ["sxt-aretomo"], }, new_connection=True, ) From 80cc5e517548061d27c7f610c5a733bd1a3c306b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 13 May 2026 10:27:44 +0100 Subject: [PATCH 03/14] Do not look for non-existant files --- src/murfey/client/context.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 73172e62c..8769f41a4 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -76,6 +76,7 @@ def ensure_dcg_exists( token: str, ) -> str | None: """Create a data collection group""" + session_file: Path | None = None if collection_type == "tomo": experiment_type_id = 36 session_file = metadata_source / "Session.dm" @@ -97,17 +98,17 @@ def ensure_dcg_exists( logger.warning(f"Get EPU session hook failed: {e}") elif collection_type == "sxt": experiment_type_id = 47 - session_file = metadata_source / "Session.dm" source_visit_dir = metadata_source.parent else: logger.error(f"Unknown collection type {collection_type}") return None - if not session_file.is_file(): - logger.warning(f"Cannot find session file {str(session_file)}") - dcg_tag = "/".join( - [part for part in metadata_source.parts if part != environment.visit] - ).replace("//", "/") + dcg_tag = "/".join( + [part for part in metadata_source.parts if part != environment.visit] + ).replace("//", "/") + if session_file is None or not session_file.is_file(): + if session_file is not None: + logger.warning(f"Cannot find session file {str(session_file)}") dcg_data = { "experiment_type_id": experiment_type_id, "tag": dcg_tag, @@ -144,10 +145,6 @@ def ensure_dcg_exists( f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}" ) - dcg_tag = "/".join( - [part for part in metadata_source.parts if part != environment.visit] - ).replace("//", "/") - for p in partial_path.split("/"): if p.startswith("Sample"): sample = int(p.replace("Sample", "")) From cb35712049237379d1640a1e86b6c9fba1b589e7 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 13 May 2026 11:13:10 +0100 Subject: [PATCH 04/14] Handle case where we want visit left in --- src/murfey/client/context.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 8769f41a4..ccb363ced 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -103,12 +103,17 @@ def ensure_dcg_exists( logger.error(f"Unknown collection type {collection_type}") return None - dcg_tag = "/".join( - [part for part in metadata_source.parts if part != environment.visit] - ).replace("//", "/") - if session_file is None or not session_file.is_file(): - if session_file is not None: - logger.warning(f"Cannot find session file {str(session_file)}") + if session_file is None: + dcg_data = { + "experiment_type_id": experiment_type_id, + "tag": metadata_source, + } + elif not session_file.is_file(): + logger.warning(f"Cannot find session file {str(session_file)}") + dcg_tag = "/".join( + [part for part in metadata_source.parts if part != environment.visit] + ## problem + ).replace("//", "/") dcg_data = { "experiment_type_id": experiment_type_id, "tag": dcg_tag, @@ -156,6 +161,11 @@ def ensure_dcg_exists( atlas=Path(partial_path), sample=sample ) + dcg_tag = "/".join( + [part for part in metadata_source.parts if part != environment.visit] + ## problem + ).replace("//", "/") + if atlas_xml_search := list( (source_visit_dir / partial_path).parent.glob("Atlas_*.xml") ): From 29e1cc218948544698312fa3f2591fdc96ed1873 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 13 May 2026 11:14:33 +0100 Subject: [PATCH 05/14] Extra send components --- src/murfey/client/contexts/sxt.py | 17 ++++++++++------- .../workflows/sxt/process_sxt_tilt_series.py | 3 +-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 60edf5228..bff876d1f 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -46,12 +46,9 @@ def register_sxt_data_collection( ) return try: - metadata_source = ( - self._basepath.parent / environment.visit / self._basepath.name - ) ensure_dcg_exists( collection_type="sxt", - metadata_source=metadata_source, + metadata_source=self._basepath, environment=environment, machine_config=self._machine_config, token=self._token, @@ -120,7 +117,11 @@ def post_transfer( data_suffixes = [".txrm"] - if transferred_file.suffix in data_suffixes and environment: + if ( + transferred_file.name.startswith("tomo") + and transferred_file.suffix in data_suffixes + and environment + ): source = _get_source(transferred_file, environment) if not source: logger.warning(f"No source found for file {transferred_file}") @@ -197,7 +198,7 @@ def post_transfer( strict=True, ) if tilt_count_txrm: - metadata["tilt_count"] = tilt_count_txrm[0] + metadata["tilt_series_length"] = tilt_count_txrm[0] self.register_sxt_data_collection( tilt_series=transferred_file.stem, @@ -227,11 +228,13 @@ def post_transfer( visit_name=environment.visit, session_id=environment.murfey_session, data={ - "session_id": environment.murfey_session, "tag": transferred_file.stem, "source": str(transferred_file.parent), "pixel_size": metadata.get("pixel_size", 100), "tilt_offset": midpoint(angles), + "tilt_series_length": metadata.get( + "tilt_series_length", len(angles) + ), "txrm": str(file_transferred_to), }, ) diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index 04674822c..adb6191f8 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -22,7 +22,6 @@ class SXTTiltSeriesInfo(BaseModel): - session_id: int tag: str source: str txrm: str @@ -39,7 +38,7 @@ def process_sxt_tilt_series_workflow( ): tilt_series_query = murfey_db.exec( select(TiltSeries) - .where(TiltSeries.session_id == tilt_series_info.session_id) + .where(TiltSeries.session_id == session_id) .where(TiltSeries.tag == tilt_series_info.tag) .where(TiltSeries.rsync_source == tilt_series_info.source) ).all() From 5657ee2da675692c3a5fbd0f941fd731e1445efb Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 14 May 2026 13:45:18 +0100 Subject: [PATCH 06/14] Switch from txrm2tiff to olefile directly --- pyproject.toml | 3 +- src/murfey/client/context.py | 10 ++- src/murfey/client/contexts/sxt.py | 107 ++++++++++++------------------ 3 files changed, 50 insertions(+), 70 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a88866ebe..01a295c7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ smartem = [ "smartem-decisions[backend]", ] sxt = [ - "txrm2tiff", + "numpy<3", + "olefile", ] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues" diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index ccb363ced..99a233aa8 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -112,7 +112,6 @@ def ensure_dcg_exists( logger.warning(f"Cannot find session file {str(session_file)}") dcg_tag = "/".join( [part for part in metadata_source.parts if part != environment.visit] - ## problem ).replace("//", "/") dcg_data = { "experiment_type_id": experiment_type_id, @@ -150,6 +149,10 @@ def ensure_dcg_exists( f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}" ) + dcg_tag = "/".join( + [part for part in metadata_source.parts if part != environment.visit] + ).replace("//", "/") + for p in partial_path.split("/"): if p.startswith("Sample"): sample = int(p.replace("Sample", "")) @@ -161,11 +164,6 @@ def ensure_dcg_exists( atlas=Path(partial_path), sample=sample ) - dcg_tag = "/".join( - [part for part in metadata_source.parts if part != environment.visit] - ## problem - ).replace("//", "/") - if atlas_xml_search := list( (source_visit_dir / partial_path).parent.glob("Atlas_*.xml") ): diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index bff876d1f..f056c653d 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -2,10 +2,8 @@ from pathlib import Path from typing import Any -from txrm2tiff.inspector import Inspector -from txrm2tiff.txrm import open_txrm -from txrm2tiff.txrm_functions.general import read_stream -from txrm2tiff.xradia_properties.enums import XrmDataTypes +import numpy as np +from olefile import OleFileIO from murfey.client.context import ( Context, @@ -117,89 +115,72 @@ def post_transfer( data_suffixes = [".txrm"] - if ( - transferred_file.name.startswith("tomo") - and transferred_file.suffix in data_suffixes - and environment - ): + if transferred_file.suffix in data_suffixes and environment: source = _get_source(transferred_file, environment) if not source: logger.warning(f"No source found for file {transferred_file}") return False # Read the tilt angles and pixel size from the txrm - metadata = { + metadata: dict[str, Any] = { "source": str(self._basepath), "tilt_series_tag": transferred_file.stem, } - with open_txrm( - transferred_file, load_images=False, load_reference=False, strict=False - ) as txrm: - inspector = Inspector(txrm) - angles = read_stream( - inspector.txrm.ole, - "ImageInfo/Angles", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if angles: + with OleFileIO(str(transferred_file)) as txrm_ole: + if txrm_ole.exists("ReferenceData/Image"): + metadata["has_reference"] = True + + if txrm_ole.exists("ImageInfo/Angles"): + angles = np.frombuffer( + txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32 + ).tolist() metadata["minimum_angle"] = min(angles) metadata["maximum_angle"] = max(angles) - pixel_size_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/PixelSize", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if pixel_size_txrm: + if txrm_ole.exists("ImageInfo/PixelSize"): + pixel_size_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/PixelSize").getvalue(), + np.float32, + ).tolist() metadata["pixel_size"] = pixel_size_txrm[0] * 1e4 - image_width_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImageWidth", - XrmDataTypes.XRM_INT, - strict=True, - ) - if image_width_txrm: + if txrm_ole.exists("ImageInfo/ImageWidth"): + image_width_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32 + ).tolist() metadata["image_size_x"] = image_width_txrm[0] - image_height_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImageHeight", - XrmDataTypes.XRM_INT, - strict=True, - ) - if image_height_txrm: + if txrm_ole.exists("ImageInfo/ImageHeight"): + image_height_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(), + np.int32, + ).tolist() metadata["image_size_y"] = image_height_txrm[0] - exposure_time_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ExpTimes", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if exposure_time_txrm: + if txrm_ole.exists("ImageInfo/ExpTimes"): + exposure_time_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32 + ).tolist() metadata["exposure_time"] = exposure_time_txrm[0] - magnification_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/XrayMagnification", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if magnification_txrm: + if txrm_ole.exists("ImageInfo/XrayMagnification"): + magnification_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(), + np.float32, + ).tolist() metadata["magnification"] = magnification_txrm[0] - tilt_count_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImagesTaken", - XrmDataTypes.XRM_INT, - strict=True, - ) - if tilt_count_txrm: + if txrm_ole.exists("ImageInfo/ImagesTaken"): + tilt_count_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(), + np.int32, + ).tolist() metadata["tilt_series_length"] = tilt_count_txrm[0] + if not metadata.get("has_reference", False): + logger.debug(f"Reference image {transferred_file} not processed") + return True + self.register_sxt_data_collection( tilt_series=transferred_file.stem, data_collection_parameters=metadata, From dc6aa4aad7231e226fe7bcabd0d0fde35c3858ce Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 14 May 2026 14:10:11 +0100 Subject: [PATCH 07/14] Dcg tag is needed --- src/murfey/client/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 99a233aa8..bded966d8 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -104,9 +104,10 @@ def ensure_dcg_exists( return None if session_file is None: + dcg_tag = "/".join(metadata_source.parts).replace("//", "/") dcg_data = { "experiment_type_id": experiment_type_id, - "tag": metadata_source, + "tag": dcg_tag, } elif not session_file.is_file(): logger.warning(f"Cannot find session file {str(session_file)}") From 97477b41bce268ba22bfce5e1e9c3bd528beeed1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 15 May 2026 16:00:48 +0100 Subject: [PATCH 08/14] Read energy and insert into ispyb --- src/murfey/client/contexts/sxt.py | 32 +++++++++++++++++-- src/murfey/server/api/workflow.py | 2 ++ .../workflows/register_data_collection.py | 7 +++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index f056c653d..c1777df95 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -61,11 +61,12 @@ def register_sxt_data_collection( "source": str(self._basepath), "tag": tilt_series, "pixel_size_on_image": str( - data_collection_parameters.get("pixel_size", 100) + data_collection_parameters.get("pixel_size", 100) * 1e-10 ), "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), "magnification": data_collection_parameters.get("magnification", 0), + "energy": data_collection_parameters.get("energy", 0), "voltage": 0, } capture_post( @@ -177,16 +178,43 @@ def post_transfer( ).tolist() metadata["tilt_series_length"] = tilt_count_txrm[0] + if txrm_ole.exists("PositionInfo/AxisNames") and txrm_ole.exists( + "PositionInfo/MotorPositions" + ): + # The ImageInfo/Energy field is empty + # Instead it needs extracting from the PositionInfo list + axis_names = [ + i + for i in txrm_ole.openstream("PositionInfo/AxisNames") + .read() + .decode("ascii") + .split("\x00") + if i + ] + axis_values = np.frombuffer( + txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(), + np.float32, + ) + if "Energy" in axis_names: + energy_index = list(np.array(axis_names) == "Energy").index( + True + ) + metadata["energy"] = int(round(axis_values[energy_index])) + if not metadata.get("has_reference", False): logger.debug(f"Reference image {transferred_file} not processed") return True + visit_index = transferred_file.parent.parts.index(environment.visit) + destination_search_dir = "/".join( + transferred_file.parent.parts[: visit_index + 2] + ) self.register_sxt_data_collection( tilt_series=transferred_file.stem, data_collection_parameters=metadata, file_extension=transferred_file.suffix, image_directory=environment.default_destinations.get( - transferred_file.parent, transferred_file.parent + Path(destination_search_dir), destination_search_dir ), environment=environment, ) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index ce63b6e84..35c7272ae 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -301,6 +301,7 @@ class DCParameters(BaseModel): exposure_time: Optional[float] = None slit_width: Optional[float] = None phase_plate: bool = False + energy: float = 0 data_collection_tag: str = "" @@ -327,6 +328,7 @@ def start_dc( "image_directory": str(rsync_basepath / dc_params.image_directory), "start_time": str(datetime.now()), "voltage": dc_params.voltage, + "energy": dc_params.energy, "pixel_size": str(float(dc_params.pixel_size_on_image) * 1e9), "image_suffix": dc_params.file_extension, "experiment_type": dc_params.experiment_type, diff --git a/src/murfey/workflows/register_data_collection.py b/src/murfey/workflows/register_data_collection.py index bedaf63d5..98c21f780 100644 --- a/src/murfey/workflows/register_data_collection.py +++ b/src/murfey/workflows/register_data_collection.py @@ -65,6 +65,7 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: imageDirectory=message["image_directory"], imageSuffix=message["image_suffix"], voltage=message["voltage"], + wavelength=message["energy"], dataCollectionGroupId=dcgid, pixelSizeOnImage=message["pixel_size"], imageSizeX=message["image_size_x"], @@ -81,7 +82,11 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: tag=( message.get("tag") if message["experiment_type"] == "tomography" - else "" + else ( + message.get("tag", "").split("_angle")[0] + if message["experiment_type"] == "sxt" + else "" + ) ), ).get("return_value", None) if dcid is None: From 1d457bf01e8d04250c4a0503b51f5de359bf5bff Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 18 May 2026 09:10:44 +0100 Subject: [PATCH 09/14] For multi-layer folders need to search destination higher up --- src/murfey/client/contexts/sxt.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index c1777df95..6023cdd93 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -208,13 +208,18 @@ def post_transfer( visit_index = transferred_file.parent.parts.index(environment.visit) destination_search_dir = "/".join( transferred_file.parent.parts[: visit_index + 2] - ) + ).replace("//", "/") self.register_sxt_data_collection( tilt_series=transferred_file.stem, data_collection_parameters=metadata, file_extension=transferred_file.suffix, - image_directory=environment.default_destinations.get( - Path(destination_search_dir), destination_search_dir + image_directory=str( + Path( + environment.default_destinations.get( + Path(destination_search_dir), destination_search_dir + ) + ) + / transferred_file.parent.relative_to(destination_search_dir) ), environment=environment, ) @@ -238,7 +243,7 @@ def post_transfer( session_id=environment.murfey_session, data={ "tag": transferred_file.stem, - "source": str(transferred_file.parent), + "source": destination_search_dir, "pixel_size": metadata.get("pixel_size", 100), "tilt_offset": midpoint(angles), "tilt_series_length": metadata.get( From b86b6cbca50c841368e7fe8e19f62fe6268e62ed Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 18 May 2026 10:43:00 +0100 Subject: [PATCH 10/14] Update tests to avoid txrm2tiff --- src/murfey/client/contexts/sxt.py | 4 +- tests/client/contexts/test_sxt.py | 91 ++++++++++++------- .../test_register_data_collection.py | 1 + 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 6023cdd93..dfc53d520 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -61,7 +61,7 @@ def register_sxt_data_collection( "source": str(self._basepath), "tag": tilt_series, "pixel_size_on_image": str( - data_collection_parameters.get("pixel_size", 100) * 1e-10 + round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10 ), "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), @@ -244,7 +244,7 @@ def post_transfer( data={ "tag": transferred_file.stem, "source": destination_search_dir, - "pixel_size": metadata.get("pixel_size", 100), + "pixel_size": round(metadata.get("pixel_size", 100), 2), "tilt_offset": midpoint(angles), "tilt_series_length": metadata.get( "tilt_series_length", len(angles) diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py index c1de8f1ec..3c482e046 100644 --- a/tests/client/contexts/test_sxt.py +++ b/tests/client/contexts/test_sxt.py @@ -1,6 +1,8 @@ from unittest.mock import patch from urllib.parse import urlparse +import numpy as np + from murfey.client.contexts.sxt import SXTContext from murfey.client.instance_environment import MurfeyInstanceEnvironment @@ -36,91 +38,110 @@ def test_sxt_context_xrm(mock_post, tmp_path): @patch("requests.post") -@patch("murfey.client.contexts.sxt.Inspector") -@patch("murfey.client.contexts.sxt.open_txrm") -@patch("murfey.client.contexts.sxt.read_stream") -def test_sxt_context_txrm( - mock_read_stream, mock_open_txrm, mock_inspector, mock_post, tmp_path -): +@patch("murfey.client.contexts.sxt.OleFileIO") +def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): mock_post().status_code = 200 - mock_read_stream.side_effect = [ - [-55, -25, 5, 35, 65], # Angles - [0.01001], # Pixel size - [1024], # Image Width - [2048], # Image Height - [1.5], # Exposure time - [1000], # Mag - [5], # Image count + mock_ole_file().__enter__().exists.return_value = True + # Motor position names + mock_ole_file().__enter__().openstream().read.return_value = ( + "\x00Val1\x00\x00Energy\x00".encode() + ) + # Metadata encoded arrays + mock_ole_file().__enter__().openstream().getvalue.side_effect = [ + np.array([-55, -25, 5, 35, 65], dtype=np.float32).tobytes(), # Angles + np.array([0.01001], dtype=np.float32).tobytes(), # Pixel size + np.array([1024], dtype=np.int32).tobytes(), # Image Width + np.array([2048], dtype=np.int32).tobytes(), # Image Height + np.array([1.5], dtype=np.float32).tobytes(), # Exposure time + np.array([1000], dtype=np.float32).tobytes(), # Mag + np.array([5], dtype=np.int32).tobytes(), # Image count + np.array([0, 519, 2, 3], dtype=np.float32).tobytes(), # Motor Pos (energy) ] env = MurfeyInstanceEnvironment( url=urlparse("http://localhost:8000"), client_id=0, - sources=[tmp_path], - default_destinations={tmp_path: str(tmp_path / "destination")}, + sources=[tmp_path / "cm12345-6/grid1"], + default_destinations={ + f"{tmp_path}/cm12345-6/grid1": f"{tmp_path}/destination/cm12345-6/grid1" + }, instrument_name="", - visit="test", + visit="cm12345-6", murfey_session=1, ) - context = SXTContext("zeiss", tmp_path, {}, "") + context = SXTContext("zeiss", tmp_path / "cm12345-6/grid1", {}, "") context.post_transfer( - tmp_path / "example.txrm", + tmp_path / "cm12345-6/grid1/example.txrm", required_position_files=[], required_strings=["fractions"], environment=env, ) - mock_open_txrm.assert_called_once_with( - tmp_path / "example.txrm", load_images=False, load_reference=False, strict=False - ) - mock_inspector.assert_called_once() + mock_ole_file.assert_any_call(str(tmp_path / "cm12345-6/grid1/example.txrm")) + assert mock_ole_file().__enter__().exists.call_count == 10 + assert mock_ole_file().__enter__().openstream.call_count == 11 # 9 + 2 above + mock_ole_file().__enter__().exists.assert_any_call("ReferenceData/Image") + for field_name in [ + "ImageInfo/Angles", + "ImageInfo/PixelSize", + "ImageInfo/ImageWidth", + "ImageInfo/ImageHeight", + "ImageInfo/ExpTimes", + "ImageInfo/XrayMagnification", + "ImageInfo/ImagesTaken", + "PositionInfo/AxisNames", + "PositionInfo/MotorPositions", + ]: + mock_ole_file().__enter__().exists.assert_any_call(field_name) + mock_ole_file().__enter__().openstream.assert_any_call(field_name) assert mock_post.call_count == 5 mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/register_data_collection_group", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/register_data_collection_group", json={ "experiment_type_id": 47, - "tag": str(tmp_path), + "tag": f"{tmp_path}/cm12345-6/grid1", }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/start_data_collection", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/start_data_collection", json={ "experiment_type": "sxt", "file_extension": ".txrm", "acquisition_software": "zeiss", - "image_directory": f"{tmp_path}/destination", + "image_directory": f"{tmp_path}/destination/cm12345-6/grid1", "data_collection_tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "tag": "example", - "pixel_size_on_image": "100.1", + "pixel_size_on_image": str(100.1 * 1e-10), "image_size_x": 1024, "image_size_y": 2048, "magnification": 1000, + "energy": 519, "voltage": 0, }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/register_processing_job", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/register_processing_job", json={ "tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "recipe": "sxt-aretomo", "experiment_type": "sxt", }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/sxt/visits/test/sessions/1/sxt_tilt_series", + "http://localhost:8000/workflow/sxt/visits/cm12345-6/sessions/1/sxt_tilt_series", json={ - "session_id": 1, "tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "pixel_size": 100.1, "tilt_offset": 5, - "txrm": str(tmp_path / "destination/example.txrm"), + "tilt_series_length": 5, + "txrm": str(tmp_path / "destination/cm12345-6/grid1/example.txrm"), }, headers={"Authorization": "Bearer "}, ) diff --git a/tests/workflows/test_register_data_collection.py b/tests/workflows/test_register_data_collection.py index 88aec006c..b910a4e54 100644 --- a/tests/workflows/test_register_data_collection.py +++ b/tests/workflows/test_register_data_collection.py @@ -76,6 +76,7 @@ def test_run( "experiment_type": "SPA", "image_suffix": ".jpg", "voltage": 200, + "energy": 520, "pixel_size": 1e-9, "image_size_x": 2048, "image_size_y": 2048, From 03c6786b81b38440ee624c3ac3c72251e2df17b4 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 18 May 2026 14:40:44 +0100 Subject: [PATCH 11/14] Insert tilt series start, end, length into ispyb --- src/murfey/client/contexts/sxt.py | 7 ++++++- src/murfey/server/api/workflow.py | 16 +++++++++++----- src/murfey/workflows/register_data_collection.py | 3 +++ tests/client/contexts/test_sxt.py | 3 +++ tests/workflows/test_register_data_collection.py | 3 +++ 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index dfc53d520..3db5d09bc 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -66,8 +66,13 @@ def register_sxt_data_collection( "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), "magnification": data_collection_parameters.get("magnification", 0), - "energy": data_collection_parameters.get("energy", 0), + "energy": data_collection_parameters.get("energy"), "voltage": 0, + "axis_start": data_collection_parameters.get("minimum_angle"), + "axis_end": data_collection_parameters.get("maximum_angle"), + "tilt_series_length": data_collection_parameters.get( + "tilt_series_length" + ), } capture_post( base_url=str(environment.url.geturl()), diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 4b79f84b0..6488e2029 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -290,12 +290,15 @@ class DCParameters(BaseModel): tag: str source: str magnification: float - total_exposed_dose: Optional[float] = None - c2aperture: Optional[float] = None - exposure_time: Optional[float] = None - slit_width: Optional[float] = None + total_exposed_dose: float | None = None + c2aperture: float | None = None + exposure_time: float | None = None + slit_width: float | None = None phase_plate: bool = False - energy: float = 0 + energy: float | None = None + axis_start: float | None = None + axis_end: float | None = None + tilt_series_length: int | None = None data_collection_tag: str = "" @@ -337,6 +340,9 @@ def start_dc( "exposure_time": dc_params.exposure_time, "slit_width": dc_params.slit_width, "phase_plate": dc_params.phase_plate, + "axis_start": dc_params.axis_start, + "axis_end": dc_params.axis_end, + "tilt_series_length": dc_params.tilt_series_length, "session_id": session_id, } diff --git a/src/murfey/workflows/register_data_collection.py b/src/murfey/workflows/register_data_collection.py index 98c21f780..0c61ba5d5 100644 --- a/src/murfey/workflows/register_data_collection.py +++ b/src/murfey/workflows/register_data_collection.py @@ -76,6 +76,9 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: totalExposedDose=message.get("total_exposed_dose"), c2aperture=message.get("c2aperture"), phasePlate=int(message.get("phase_plate", 0)), + axisStart=message.get("axis_start"), + axisEnd=message.get("axis_end"), + numberOfImages=message.get("tilt_series_length"), ) dcid = _transport_object.do_insert_data_collection( record, diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py index 3c482e046..e24ff4b52 100644 --- a/tests/client/contexts/test_sxt.py +++ b/tests/client/contexts/test_sxt.py @@ -120,6 +120,9 @@ def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): "magnification": 1000, "energy": 519, "voltage": 0, + "axis_start": -55, + "axis_end": 65, + "tilt_series_length": 5, }, headers={"Authorization": "Bearer "}, ) diff --git a/tests/workflows/test_register_data_collection.py b/tests/workflows/test_register_data_collection.py index b910a4e54..9b792c126 100644 --- a/tests/workflows/test_register_data_collection.py +++ b/tests/workflows/test_register_data_collection.py @@ -86,6 +86,9 @@ def test_run( "total_exposed_dose": 30, "c2aperture": 5, "phase_plate": 1, + "axis_start": -60, + "axis_end": 55, + "tilt_series_length": 250, } result = run(message=message, murfey_db=mock_murfey_db) if dcg_result is None: From 692be6eb877d6bb35c8511a8e35075aa35d959da Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 19 May 2026 09:34:55 +0100 Subject: [PATCH 12/14] Do not reprocess if processing already requested --- src/murfey/workflows/sxt/process_sxt_tilt_series.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index adb6191f8..1c3daed0b 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -44,6 +44,9 @@ def process_sxt_tilt_series_workflow( ).all() if tilt_series_query: tilt_series = tilt_series_query[0] + if tilt_series.processing_requested: + logger.info(f"Tilt series {tilt_series.tag} has already been processed") + return else: tilt_series = TiltSeries( session_id=session_id, From 64a85ebd73c725ff855ccdbf63d721c4fb5bfa54 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 19 May 2026 09:37:05 +0100 Subject: [PATCH 13/14] Only mark processing requested once message sent --- src/murfey/workflows/sxt/process_sxt_tilt_series.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index 1c3daed0b..67009cbd0 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -53,7 +53,7 @@ def process_sxt_tilt_series_workflow( tag=tilt_series_info.tag, rsync_source=tilt_series_info.source, tilt_series_length=tilt_series_info.tilt_series_length, - processing_requested=True, + processing_requested=False, ) murfey_db.add(tilt_series) murfey_db.commit() @@ -116,3 +116,6 @@ def process_sxt_tilt_series_workflow( logger.info( f"No transport object found. Zocalo message would be {sanitise(str(zocalo_message))}" ) + tilt_series.processing_requested = True + murfey_db.add(tilt_series) + murfey_db.commit() From e23ed2298c75e9ba56698044e07d98d6034e8221 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 19 May 2026 10:09:26 +0100 Subject: [PATCH 14/14] Comment pixel units --- src/murfey/client/contexts/sxt.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 3db5d09bc..b65b94499 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -62,7 +62,7 @@ def register_sxt_data_collection( "tag": tilt_series, "pixel_size_on_image": str( round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10 - ), + ), # expected in metres "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), "magnification": data_collection_parameters.get("magnification", 0), @@ -212,7 +212,7 @@ def post_transfer( visit_index = transferred_file.parent.parts.index(environment.visit) destination_search_dir = "/".join( - transferred_file.parent.parts[: visit_index + 2] + transferred_file.parts[: visit_index + 2] ).replace("//", "/") self.register_sxt_data_collection( tilt_series=transferred_file.stem, @@ -249,7 +249,9 @@ def post_transfer( data={ "tag": transferred_file.stem, "source": destination_search_dir, - "pixel_size": round(metadata.get("pixel_size", 100), 2), + "pixel_size": round( + metadata.get("pixel_size", 100), 2 + ), # angstroms "tilt_offset": midpoint(angles), "tilt_series_length": metadata.get( "tilt_series_length", len(angles)