SXT bugfixes and improvements

pyproject.toml

@@ -72,7 +72,8 @@ smartem = [
     "smartem-decisions[backend]",
 ]
 sxt = [
-    "txrm2tiff",
+    "numpy<3",
+    "olefile",
 ]
 [project.urls]
 Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues"

src/murfey/client/context.py

@@ -76,6 +76,7 @@ def ensure_dcg_exists(
     token: str,
 ) -> str | None:
     """Create  a data collection group"""
+    session_file: Path | None = None
     if collection_type == "tomo":
         experiment_type_id = 36
         session_file = metadata_source / "Session.dm"
@@ -97,13 +98,18 @@ def ensure_dcg_exists(
                 logger.warning(f"Get EPU session hook failed: {e}")
     elif collection_type == "sxt":
         experiment_type_id = 47
-        session_file = metadata_source / "Session.dm"
         source_visit_dir = metadata_source.parent
     else:
         logger.error(f"Unknown collection type {collection_type}")
         return None
 
-    if not session_file.is_file():
+    if session_file is None:
+        dcg_tag = "/".join(metadata_source.parts).replace("//", "/")
+        dcg_data = {
+            "experiment_type_id": experiment_type_id,
+            "tag": dcg_tag,
+        }
+    elif not session_file.is_file():
         logger.warning(f"Cannot find session file {str(session_file)}")
         dcg_tag = "/".join(
             [part for part in metadata_source.parts if part != environment.visit]

src/murfey/client/contexts/sxt.py

@@ -2,10 +2,8 @@
 from pathlib import Path
 from typing import Any
 
-from txrm2tiff.inspector import Inspector
-from txrm2tiff.txrm import open_txrm
-from txrm2tiff.txrm_functions.general import read_stream
-from txrm2tiff.xradia_properties.enums import XrmDataTypes
+import numpy as np
+from olefile import OleFileIO
 
 from murfey.client.context import (
     Context,
@@ -46,12 +44,9 @@ def register_sxt_data_collection(
             )
             return
         try:
-            metadata_source = (
-                self._basepath.parent / environment.visit / self._basepath.name
-            )
             ensure_dcg_exists(
                 collection_type="sxt",
-                metadata_source=metadata_source,
+                metadata_source=self._basepath,
                 environment=environment,
                 machine_config=self._machine_config,
                 token=self._token,
@@ -66,12 +61,18 @@ def register_sxt_data_collection(
                 "source": str(self._basepath),
                 "tag": tilt_series,
                 "pixel_size_on_image": str(
-                    data_collection_parameters.get("pixel_size", 100)
-                ),
+                    round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10
+                ),  # expected in metres
                 "image_size_x": data_collection_parameters.get("image_size_x", 0),
                 "image_size_y": data_collection_parameters.get("image_size_y", 0),
                 "magnification": data_collection_parameters.get("magnification", 0),
+                "energy": data_collection_parameters.get("energy"),
                 "voltage": 0,
+                "axis_start": data_collection_parameters.get("minimum_angle"),
+                "axis_end": data_collection_parameters.get("maximum_angle"),
+                "tilt_series_length": data_collection_parameters.get(
+                    "tilt_series_length"
+                ),
             }
             capture_post(
                 base_url=str(environment.url.geturl()),
@@ -127,84 +128,103 @@ def post_transfer(
                 return False
 
             # Read the tilt angles and pixel size from the txrm
-            metadata = {
+            metadata: dict[str, Any] = {
                 "source": str(self._basepath),
                 "tilt_series_tag": transferred_file.stem,
             }
-            with open_txrm(
-                transferred_file, load_images=False, load_reference=False, strict=False
-            ) as txrm:
-                inspector = Inspector(txrm)
-                angles = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/Angles",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if angles:
+            with OleFileIO(str(transferred_file)) as txrm_ole:
+                if txrm_ole.exists("ReferenceData/Image"):
+                    metadata["has_reference"] = True
+
+                if txrm_ole.exists("ImageInfo/Angles"):
+                    angles = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32
+                    ).tolist()
                     metadata["minimum_angle"] = min(angles)
                     metadata["maximum_angle"] = max(angles)
 
-                pixel_size_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/PixelSize",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if pixel_size_txrm:
+                if txrm_ole.exists("ImageInfo/PixelSize"):
+                    pixel_size_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/PixelSize").getvalue(),
+                        np.float32,
+                    ).tolist()
                     metadata["pixel_size"] = pixel_size_txrm[0] * 1e4
 
-                image_width_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImageWidth",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if image_width_txrm:
+                if txrm_ole.exists("ImageInfo/ImageWidth"):
+                    image_width_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32
+                    ).tolist()
                     metadata["image_size_x"] = image_width_txrm[0]
 
-                image_height_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImageHeight",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if image_height_txrm:
+                if txrm_ole.exists("ImageInfo/ImageHeight"):
+                    image_height_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(),
+                        np.int32,
+                    ).tolist()
                     metadata["image_size_y"] = image_height_txrm[0]
 
-                exposure_time_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ExpTimes",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if exposure_time_txrm:
+                if txrm_ole.exists("ImageInfo/ExpTimes"):
+                    exposure_time_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32
+                    ).tolist()
                     metadata["exposure_time"] = exposure_time_txrm[0]
 
-                magnification_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/XrayMagnification",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if magnification_txrm:
+                if txrm_ole.exists("ImageInfo/XrayMagnification"):
+                    magnification_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(),
+                        np.float32,
+                    ).tolist()
                     metadata["magnification"] = magnification_txrm[0]
 
-                tilt_count_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImagesTaken",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if tilt_count_txrm:
-                    metadata["tilt_count"] = tilt_count_txrm[0]
+                if txrm_ole.exists("ImageInfo/ImagesTaken"):
+                    tilt_count_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(),
+                        np.int32,
+                    ).tolist()
+                    metadata["tilt_series_length"] = tilt_count_txrm[0]
+
+                if txrm_ole.exists("PositionInfo/AxisNames") and txrm_ole.exists(
+                    "PositionInfo/MotorPositions"
+                ):
+                    # The ImageInfo/Energy field is empty
+                    # Instead it needs extracting from the PositionInfo list
+                    axis_names = [
+                        i
+                        for i in txrm_ole.openstream("PositionInfo/AxisNames")
+                        .read()
+                        .decode("ascii")
+                        .split("\x00")
+                        if i
+                    ]
+                    axis_values = np.frombuffer(
+                        txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(),
+                        np.float32,
+                    )
+                    if "Energy" in axis_names:
+                        energy_index = list(np.array(axis_names) == "Energy").index(
+                            True
+                        )
+                        metadata["energy"] = int(round(axis_values[energy_index]))
+
+            if not metadata.get("has_reference", False):
+                logger.debug(f"Reference image {transferred_file} not processed")
+                return True
 
+            visit_index = transferred_file.parent.parts.index(environment.visit)
+            destination_search_dir = "/".join(
+                transferred_file.parts[: visit_index + 2]
+            ).replace("//", "/")
             self.register_sxt_data_collection(
                 tilt_series=transferred_file.stem,
                 data_collection_parameters=metadata,
                 file_extension=transferred_file.suffix,
-                image_directory=environment.default_destinations.get(
-                    transferred_file.parent, transferred_file.parent
+                image_directory=str(
+                    Path(
+                        environment.default_destinations.get(
+                            Path(destination_search_dir), destination_search_dir
+                        )
+                    )
+                    / transferred_file.parent.relative_to(destination_search_dir)
                 ),
                 environment=environment,
             )
@@ -227,11 +247,15 @@ def post_transfer(
                 visit_name=environment.visit,
                 session_id=environment.murfey_session,
                 data={
-                    "session_id": environment.murfey_session,
                     "tag": transferred_file.stem,
-                    "source": str(transferred_file.parent),
-                    "pixel_size": metadata.get("pixel_size", 100),
+                    "source": destination_search_dir,
+                    "pixel_size": round(
+                        metadata.get("pixel_size", 100), 2
+                    ),  # angstroms
                     "tilt_offset": midpoint(angles),
+                    "tilt_series_length": metadata.get(
+                        "tilt_series_length", len(angles)
+                    ),
                     "txrm": str(file_transferred_to),
                 },
             )

src/murfey/server/api/workflow.py

@@ -290,11 +290,15 @@ class DCParameters(BaseModel):
     tag: str
     source: str
     magnification: float
-    total_exposed_dose: Optional[float] = None
-    c2aperture: Optional[float] = None
-    exposure_time: Optional[float] = None
-    slit_width: Optional[float] = None
+    total_exposed_dose: float | None = None
+    c2aperture: float | None = None
+    exposure_time: float | None = None
+    slit_width: float | None = None
     phase_plate: bool = False
+    energy: float | None = None
+    axis_start: float | None = None
+    axis_end: float | None = None
+    tilt_series_length: int | None = None
     data_collection_tag: str = ""
 
 
@@ -321,6 +325,7 @@ def start_dc(
         "image_directory": str(rsync_basepath / dc_params.image_directory),
         "start_time": str(datetime.now()),
         "voltage": dc_params.voltage,
+        "energy": dc_params.energy,
         "pixel_size": str(float(dc_params.pixel_size_on_image) * 1e9),
         "image_suffix": dc_params.file_extension,
         "experiment_type": dc_params.experiment_type,
@@ -335,6 +340,9 @@ def start_dc(
         "exposure_time": dc_params.exposure_time,
         "slit_width": dc_params.slit_width,
         "phase_plate": dc_params.phase_plate,
+        "axis_start": dc_params.axis_start,
+        "axis_end": dc_params.axis_end,
+        "tilt_series_length": dc_params.tilt_series_length,
         "session_id": session_id,
     }
 

src/murfey/workflows/register_data_collection.py

@@ -65,6 +65,7 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]:
                 imageDirectory=message["image_directory"],
                 imageSuffix=message["image_suffix"],
                 voltage=message["voltage"],
+                wavelength=message["energy"],
                 dataCollectionGroupId=dcgid,
                 pixelSizeOnImage=message["pixel_size"],
                 imageSizeX=message["image_size_x"],
@@ -75,13 +76,20 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]:
                 totalExposedDose=message.get("total_exposed_dose"),
                 c2aperture=message.get("c2aperture"),
                 phasePlate=int(message.get("phase_plate", 0)),
+                axisStart=message.get("axis_start"),
+                axisEnd=message.get("axis_end"),
+                numberOfImages=message.get("tilt_series_length"),
             )
             dcid = _transport_object.do_insert_data_collection(
                 record,
                 tag=(
                     message.get("tag")
                     if message["experiment_type"] == "tomography"
-                    else ""
+                    else (
+                        message.get("tag", "").split("_angle")[0]
+                        if message["experiment_type"] == "sxt"
+                        else ""
+                    )
                 ),
             ).get("return_value", None)
             if dcid is None:

src/murfey/workflows/sxt/process_sxt_tilt_series.py

@@ -22,7 +22,6 @@
 
 
 class SXTTiltSeriesInfo(BaseModel):
-    session_id: int
     tag: str
     source: str
     txrm: str
@@ -39,19 +38,22 @@ def process_sxt_tilt_series_workflow(
 ):
     tilt_series_query = murfey_db.exec(
         select(TiltSeries)
-        .where(TiltSeries.session_id == tilt_series_info.session_id)
+        .where(TiltSeries.session_id == session_id)
         .where(TiltSeries.tag == tilt_series_info.tag)
         .where(TiltSeries.rsync_source == tilt_series_info.source)
     ).all()
     if tilt_series_query:
         tilt_series = tilt_series_query[0]
+        if tilt_series.processing_requested:
+            logger.info(f"Tilt series {tilt_series.tag} has already been processed")
+            return
     else:
         tilt_series = TiltSeries(
             session_id=session_id,
             tag=tilt_series_info.tag,
             rsync_source=tilt_series_info.source,
             tilt_series_length=tilt_series_info.tilt_series_length,
-            processing_requested=True,
+            processing_requested=False,
         )
         murfey_db.add(tilt_series)
         murfey_db.commit()
@@ -114,3 +116,6 @@ def process_sxt_tilt_series_workflow(
         logger.info(
             f"No transport object found. Zocalo message would be {sanitise(str(zocalo_message))}"
         )
+    tilt_series.processing_requested = True
+    murfey_db.add(tilt_series)
+    murfey_db.commit()