diff --git a/pyproject.toml b/pyproject.toml index a88866ebe..01a295c7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ smartem = [ "smartem-decisions[backend]", ] sxt = [ - "txrm2tiff", + "numpy<3", + "olefile", ] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues" diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 73172e62c..bded966d8 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -76,6 +76,7 @@ def ensure_dcg_exists( token: str, ) -> str | None: """Create a data collection group""" + session_file: Path | None = None if collection_type == "tomo": experiment_type_id = 36 session_file = metadata_source / "Session.dm" @@ -97,13 +98,18 @@ def ensure_dcg_exists( logger.warning(f"Get EPU session hook failed: {e}") elif collection_type == "sxt": experiment_type_id = 47 - session_file = metadata_source / "Session.dm" source_visit_dir = metadata_source.parent else: logger.error(f"Unknown collection type {collection_type}") return None - if not session_file.is_file(): + if session_file is None: + dcg_tag = "/".join(metadata_source.parts).replace("//", "/") + dcg_data = { + "experiment_type_id": experiment_type_id, + "tag": dcg_tag, + } + elif not session_file.is_file(): logger.warning(f"Cannot find session file {str(session_file)}") dcg_tag = "/".join( [part for part in metadata_source.parts if part != environment.visit] diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 60edf5228..b65b94499 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -2,10 +2,8 @@ from pathlib import Path from typing import Any -from txrm2tiff.inspector import Inspector -from txrm2tiff.txrm import open_txrm -from txrm2tiff.txrm_functions.general import read_stream -from txrm2tiff.xradia_properties.enums import XrmDataTypes +import numpy as np +from olefile import OleFileIO from murfey.client.context import ( Context, @@ -46,12 +44,9 @@ def register_sxt_data_collection( ) return try: - metadata_source = ( - self._basepath.parent / environment.visit / self._basepath.name - ) ensure_dcg_exists( collection_type="sxt", - metadata_source=metadata_source, + metadata_source=self._basepath, environment=environment, machine_config=self._machine_config, token=self._token, @@ -66,12 +61,18 @@ def register_sxt_data_collection( "source": str(self._basepath), "tag": tilt_series, "pixel_size_on_image": str( - data_collection_parameters.get("pixel_size", 100) - ), + round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10 + ), # expected in metres "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), "magnification": data_collection_parameters.get("magnification", 0), + "energy": data_collection_parameters.get("energy"), "voltage": 0, + "axis_start": data_collection_parameters.get("minimum_angle"), + "axis_end": data_collection_parameters.get("maximum_angle"), + "tilt_series_length": data_collection_parameters.get( + "tilt_series_length" + ), } capture_post( base_url=str(environment.url.geturl()), @@ -127,84 +128,103 @@ def post_transfer( return False # Read the tilt angles and pixel size from the txrm - metadata = { + metadata: dict[str, Any] = { "source": str(self._basepath), "tilt_series_tag": transferred_file.stem, } - with open_txrm( - transferred_file, load_images=False, load_reference=False, strict=False - ) as txrm: - inspector = Inspector(txrm) - angles = read_stream( - inspector.txrm.ole, - "ImageInfo/Angles", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if angles: + with OleFileIO(str(transferred_file)) as txrm_ole: + if txrm_ole.exists("ReferenceData/Image"): + metadata["has_reference"] = True + + if txrm_ole.exists("ImageInfo/Angles"): + angles = np.frombuffer( + txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32 + ).tolist() metadata["minimum_angle"] = min(angles) metadata["maximum_angle"] = max(angles) - pixel_size_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/PixelSize", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if pixel_size_txrm: + if txrm_ole.exists("ImageInfo/PixelSize"): + pixel_size_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/PixelSize").getvalue(), + np.float32, + ).tolist() metadata["pixel_size"] = pixel_size_txrm[0] * 1e4 - image_width_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImageWidth", - XrmDataTypes.XRM_INT, - strict=True, - ) - if image_width_txrm: + if txrm_ole.exists("ImageInfo/ImageWidth"): + image_width_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32 + ).tolist() metadata["image_size_x"] = image_width_txrm[0] - image_height_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImageHeight", - XrmDataTypes.XRM_INT, - strict=True, - ) - if image_height_txrm: + if txrm_ole.exists("ImageInfo/ImageHeight"): + image_height_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(), + np.int32, + ).tolist() metadata["image_size_y"] = image_height_txrm[0] - exposure_time_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ExpTimes", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if exposure_time_txrm: + if txrm_ole.exists("ImageInfo/ExpTimes"): + exposure_time_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32 + ).tolist() metadata["exposure_time"] = exposure_time_txrm[0] - magnification_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/XrayMagnification", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) - if magnification_txrm: + if txrm_ole.exists("ImageInfo/XrayMagnification"): + magnification_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(), + np.float32, + ).tolist() metadata["magnification"] = magnification_txrm[0] - tilt_count_txrm = read_stream( - inspector.txrm.ole, - "ImageInfo/ImagesTaken", - XrmDataTypes.XRM_INT, - strict=True, - ) - if tilt_count_txrm: - metadata["tilt_count"] = tilt_count_txrm[0] + if txrm_ole.exists("ImageInfo/ImagesTaken"): + tilt_count_txrm = np.frombuffer( + txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(), + np.int32, + ).tolist() + metadata["tilt_series_length"] = tilt_count_txrm[0] + + if txrm_ole.exists("PositionInfo/AxisNames") and txrm_ole.exists( + "PositionInfo/MotorPositions" + ): + # The ImageInfo/Energy field is empty + # Instead it needs extracting from the PositionInfo list + axis_names = [ + i + for i in txrm_ole.openstream("PositionInfo/AxisNames") + .read() + .decode("ascii") + .split("\x00") + if i + ] + axis_values = np.frombuffer( + txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(), + np.float32, + ) + if "Energy" in axis_names: + energy_index = list(np.array(axis_names) == "Energy").index( + True + ) + metadata["energy"] = int(round(axis_values[energy_index])) + + if not metadata.get("has_reference", False): + logger.debug(f"Reference image {transferred_file} not processed") + return True + visit_index = transferred_file.parent.parts.index(environment.visit) + destination_search_dir = "/".join( + transferred_file.parts[: visit_index + 2] + ).replace("//", "/") self.register_sxt_data_collection( tilt_series=transferred_file.stem, data_collection_parameters=metadata, file_extension=transferred_file.suffix, - image_directory=environment.default_destinations.get( - transferred_file.parent, transferred_file.parent + image_directory=str( + Path( + environment.default_destinations.get( + Path(destination_search_dir), destination_search_dir + ) + ) + / transferred_file.parent.relative_to(destination_search_dir) ), environment=environment, ) @@ -227,11 +247,15 @@ def post_transfer( visit_name=environment.visit, session_id=environment.murfey_session, data={ - "session_id": environment.murfey_session, "tag": transferred_file.stem, - "source": str(transferred_file.parent), - "pixel_size": metadata.get("pixel_size", 100), + "source": destination_search_dir, + "pixel_size": round( + metadata.get("pixel_size", 100), 2 + ), # angstroms "tilt_offset": midpoint(angles), + "tilt_series_length": metadata.get( + "tilt_series_length", len(angles) + ), "txrm": str(file_transferred_to), }, ) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 4eb7906ba..6488e2029 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -290,11 +290,15 @@ class DCParameters(BaseModel): tag: str source: str magnification: float - total_exposed_dose: Optional[float] = None - c2aperture: Optional[float] = None - exposure_time: Optional[float] = None - slit_width: Optional[float] = None + total_exposed_dose: float | None = None + c2aperture: float | None = None + exposure_time: float | None = None + slit_width: float | None = None phase_plate: bool = False + energy: float | None = None + axis_start: float | None = None + axis_end: float | None = None + tilt_series_length: int | None = None data_collection_tag: str = "" @@ -321,6 +325,7 @@ def start_dc( "image_directory": str(rsync_basepath / dc_params.image_directory), "start_time": str(datetime.now()), "voltage": dc_params.voltage, + "energy": dc_params.energy, "pixel_size": str(float(dc_params.pixel_size_on_image) * 1e9), "image_suffix": dc_params.file_extension, "experiment_type": dc_params.experiment_type, @@ -335,6 +340,9 @@ def start_dc( "exposure_time": dc_params.exposure_time, "slit_width": dc_params.slit_width, "phase_plate": dc_params.phase_plate, + "axis_start": dc_params.axis_start, + "axis_end": dc_params.axis_end, + "tilt_series_length": dc_params.tilt_series_length, "session_id": session_id, } diff --git a/src/murfey/workflows/register_data_collection.py b/src/murfey/workflows/register_data_collection.py index bedaf63d5..0c61ba5d5 100644 --- a/src/murfey/workflows/register_data_collection.py +++ b/src/murfey/workflows/register_data_collection.py @@ -65,6 +65,7 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: imageDirectory=message["image_directory"], imageSuffix=message["image_suffix"], voltage=message["voltage"], + wavelength=message["energy"], dataCollectionGroupId=dcgid, pixelSizeOnImage=message["pixel_size"], imageSizeX=message["image_size_x"], @@ -75,13 +76,20 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: totalExposedDose=message.get("total_exposed_dose"), c2aperture=message.get("c2aperture"), phasePlate=int(message.get("phase_plate", 0)), + axisStart=message.get("axis_start"), + axisEnd=message.get("axis_end"), + numberOfImages=message.get("tilt_series_length"), ) dcid = _transport_object.do_insert_data_collection( record, tag=( message.get("tag") if message["experiment_type"] == "tomography" - else "" + else ( + message.get("tag", "").split("_angle")[0] + if message["experiment_type"] == "sxt" + else "" + ) ), ).get("return_value", None) if dcid is None: diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index 04674822c..67009cbd0 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -22,7 +22,6 @@ class SXTTiltSeriesInfo(BaseModel): - session_id: int tag: str source: str txrm: str @@ -39,19 +38,22 @@ def process_sxt_tilt_series_workflow( ): tilt_series_query = murfey_db.exec( select(TiltSeries) - .where(TiltSeries.session_id == tilt_series_info.session_id) + .where(TiltSeries.session_id == session_id) .where(TiltSeries.tag == tilt_series_info.tag) .where(TiltSeries.rsync_source == tilt_series_info.source) ).all() if tilt_series_query: tilt_series = tilt_series_query[0] + if tilt_series.processing_requested: + logger.info(f"Tilt series {tilt_series.tag} has already been processed") + return else: tilt_series = TiltSeries( session_id=session_id, tag=tilt_series_info.tag, rsync_source=tilt_series_info.source, tilt_series_length=tilt_series_info.tilt_series_length, - processing_requested=True, + processing_requested=False, ) murfey_db.add(tilt_series) murfey_db.commit() @@ -114,3 +116,6 @@ def process_sxt_tilt_series_workflow( logger.info( f"No transport object found. Zocalo message would be {sanitise(str(zocalo_message))}" ) + tilt_series.processing_requested = True + murfey_db.add(tilt_series) + murfey_db.commit() diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py index c1de8f1ec..e24ff4b52 100644 --- a/tests/client/contexts/test_sxt.py +++ b/tests/client/contexts/test_sxt.py @@ -1,6 +1,8 @@ from unittest.mock import patch from urllib.parse import urlparse +import numpy as np + from murfey.client.contexts.sxt import SXTContext from murfey.client.instance_environment import MurfeyInstanceEnvironment @@ -36,91 +38,113 @@ def test_sxt_context_xrm(mock_post, tmp_path): @patch("requests.post") -@patch("murfey.client.contexts.sxt.Inspector") -@patch("murfey.client.contexts.sxt.open_txrm") -@patch("murfey.client.contexts.sxt.read_stream") -def test_sxt_context_txrm( - mock_read_stream, mock_open_txrm, mock_inspector, mock_post, tmp_path -): +@patch("murfey.client.contexts.sxt.OleFileIO") +def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): mock_post().status_code = 200 - mock_read_stream.side_effect = [ - [-55, -25, 5, 35, 65], # Angles - [0.01001], # Pixel size - [1024], # Image Width - [2048], # Image Height - [1.5], # Exposure time - [1000], # Mag - [5], # Image count + mock_ole_file().__enter__().exists.return_value = True + # Motor position names + mock_ole_file().__enter__().openstream().read.return_value = ( + "\x00Val1\x00\x00Energy\x00".encode() + ) + # Metadata encoded arrays + mock_ole_file().__enter__().openstream().getvalue.side_effect = [ + np.array([-55, -25, 5, 35, 65], dtype=np.float32).tobytes(), # Angles + np.array([0.01001], dtype=np.float32).tobytes(), # Pixel size + np.array([1024], dtype=np.int32).tobytes(), # Image Width + np.array([2048], dtype=np.int32).tobytes(), # Image Height + np.array([1.5], dtype=np.float32).tobytes(), # Exposure time + np.array([1000], dtype=np.float32).tobytes(), # Mag + np.array([5], dtype=np.int32).tobytes(), # Image count + np.array([0, 519, 2, 3], dtype=np.float32).tobytes(), # Motor Pos (energy) ] env = MurfeyInstanceEnvironment( url=urlparse("http://localhost:8000"), client_id=0, - sources=[tmp_path], - default_destinations={tmp_path: str(tmp_path / "destination")}, + sources=[tmp_path / "cm12345-6/grid1"], + default_destinations={ + f"{tmp_path}/cm12345-6/grid1": f"{tmp_path}/destination/cm12345-6/grid1" + }, instrument_name="", - visit="test", + visit="cm12345-6", murfey_session=1, ) - context = SXTContext("zeiss", tmp_path, {}, "") + context = SXTContext("zeiss", tmp_path / "cm12345-6/grid1", {}, "") context.post_transfer( - tmp_path / "example.txrm", + tmp_path / "cm12345-6/grid1/example.txrm", required_position_files=[], required_strings=["fractions"], environment=env, ) - mock_open_txrm.assert_called_once_with( - tmp_path / "example.txrm", load_images=False, load_reference=False, strict=False - ) - mock_inspector.assert_called_once() + mock_ole_file.assert_any_call(str(tmp_path / "cm12345-6/grid1/example.txrm")) + assert mock_ole_file().__enter__().exists.call_count == 10 + assert mock_ole_file().__enter__().openstream.call_count == 11 # 9 + 2 above + mock_ole_file().__enter__().exists.assert_any_call("ReferenceData/Image") + for field_name in [ + "ImageInfo/Angles", + "ImageInfo/PixelSize", + "ImageInfo/ImageWidth", + "ImageInfo/ImageHeight", + "ImageInfo/ExpTimes", + "ImageInfo/XrayMagnification", + "ImageInfo/ImagesTaken", + "PositionInfo/AxisNames", + "PositionInfo/MotorPositions", + ]: + mock_ole_file().__enter__().exists.assert_any_call(field_name) + mock_ole_file().__enter__().openstream.assert_any_call(field_name) assert mock_post.call_count == 5 mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/register_data_collection_group", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/register_data_collection_group", json={ "experiment_type_id": 47, - "tag": str(tmp_path), + "tag": f"{tmp_path}/cm12345-6/grid1", }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/start_data_collection", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/start_data_collection", json={ "experiment_type": "sxt", "file_extension": ".txrm", "acquisition_software": "zeiss", - "image_directory": f"{tmp_path}/destination", + "image_directory": f"{tmp_path}/destination/cm12345-6/grid1", "data_collection_tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "tag": "example", - "pixel_size_on_image": "100.1", + "pixel_size_on_image": str(100.1 * 1e-10), "image_size_x": 1024, "image_size_y": 2048, "magnification": 1000, + "energy": 519, "voltage": 0, + "axis_start": -55, + "axis_end": 65, + "tilt_series_length": 5, }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/visits/test/sessions/1/register_processing_job", + "http://localhost:8000/workflow/visits/cm12345-6/sessions/1/register_processing_job", json={ "tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "recipe": "sxt-aretomo", "experiment_type": "sxt", }, headers={"Authorization": "Bearer "}, ) mock_post.assert_any_call( - "http://localhost:8000/workflow/sxt/visits/test/sessions/1/sxt_tilt_series", + "http://localhost:8000/workflow/sxt/visits/cm12345-6/sessions/1/sxt_tilt_series", json={ - "session_id": 1, "tag": "example", - "source": str(tmp_path), + "source": f"{tmp_path}/cm12345-6/grid1", "pixel_size": 100.1, "tilt_offset": 5, - "txrm": str(tmp_path / "destination/example.txrm"), + "tilt_series_length": 5, + "txrm": str(tmp_path / "destination/cm12345-6/grid1/example.txrm"), }, headers={"Authorization": "Bearer "}, ) diff --git a/tests/workflows/test_register_data_collection.py b/tests/workflows/test_register_data_collection.py index 88aec006c..9b792c126 100644 --- a/tests/workflows/test_register_data_collection.py +++ b/tests/workflows/test_register_data_collection.py @@ -76,6 +76,7 @@ def test_run( "experiment_type": "SPA", "image_suffix": ".jpg", "voltage": 200, + "energy": 520, "pixel_size": 1e-9, "image_size_x": 2048, "image_size_y": 2048, @@ -85,6 +86,9 @@ def test_run( "total_exposed_dose": 30, "c2aperture": 5, "phase_plate": 1, + "axis_start": -60, + "axis_end": 55, + "tilt_series_length": 250, } result = run(message=message, murfey_db=mock_murfey_db) if dcg_result is None: