Add a real test

Add script to dump test meta messages from frameProcessor

python/setup.cfg

@@ -38,6 +38,7 @@ dev =
     flake8-isort
     pytest-cov
     pytest-asyncio
+    h5py
     sphinx-rtd-theme-github-versions
     pydata-sphinx-theme>=0.12
     sphinx-autobuild
@@ -47,6 +48,8 @@ dev =
     tox-direct
     pre-commit
     requests
+    typer
+    pyzmq
 
 [options.entry_points]
 # Include a command line script

python/tests/dump_zeromq_stream.py

@@ -0,0 +1,52 @@
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+import zmq
+
+HERE = Path(__file__).parent
+
+
+def main(
+    url: Annotated[
+        str, typer.Argument(help="URL of ZeroMQ server")
+    ] = "127.0.0.1:10008",
+    output: Annotated[Path, typer.Option(help="Output directory")] = HERE / "input",
+):
+    """Dump messages from a ZeroMQ PUB stream."""
+    if not url.startswith("http://"):
+        url = f"tcp://{url}"
+
+    context = zmq.Context()
+    socket = context.socket(zmq.SUB)
+    socket.connect(url)
+    socket.setsockopt_string(zmq.SUBSCRIBE, "")
+
+    print(f"Connected to {url}")
+
+    idx = 0
+    while True:
+        print("Waiting for message...")
+        message = socket.recv()
+        # Check if message is json or raw bytes
+        try:
+            j = json.loads(message.decode())
+            data = json.dumps(j, indent=4)
+            ext = ".json"
+            mode = "w"
+        except (json.decoder.JSONDecodeError, UnicodeDecodeError):
+            data = message
+            ext = ""
+            mode = "wb"
+
+        with (output / f"{idx}{ext}").open(mode=mode) as f:
+            f.write(data)
+
+        print(f"{idx}{ext}")
+
+        idx += 1
+
+
+if __name__ == "__main__":
+    typer.run(main)

python/tests/input/00.json

@@ -0,0 +1,10 @@
+{
+    "plugin": "Acquisition",
+    "parameter": "startacquisition",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "rank": 0,
+        "totalFrames": 1
+    }
+}

python/tests/input/02.json

@@ -0,0 +1,10 @@
+{
+    "plugin": "Acquisition",
+    "parameter": "createfile",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "rank": 0,
+        "totalFrames": 1
+    }
+}

python/tests/input/03.json

@@ -0,0 +1,4 @@
+{
+    "filePath": "/dls/i03/data/2024/cm37235-3/tmp/gary/0/test_0_0_000001.h5",
+    "create_duration": 2071
+}

python/tests/input/04.json

@@ -0,0 +1,9 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-globalconfig",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "series": 40181
+    }
+}

python/tests/input/05.json

@@ -0,0 +1,54 @@
+{
+    "auto_summation": true,
+    "beam_center_x": 2047.8194396763713,
+    "beam_center_y": 2146.4601198541245,
+    "bit_depth_image": 32,
+    "bit_depth_readout": 16,
+    "chi_increment": 0.0,
+    "chi_start": 0.0,
+    "compression": "bslz4",
+    "count_time": 0.0999999,
+    "countrate_correction_applied": true,
+    "countrate_correction_count_cutoff": 1151122,
+    "data_collection_date": "2024-05-25T08:03:25.903+02:00",
+    "description": "Dectris EIGER2 Si 16M",
+    "detector_distance": 376.5651820410446,
+    "detector_number": "E-32-0117",
+    "detector_readout_time": 1e-07,
+    "detector_translation": [
+        0.15358645797572784,
+        0.16098450898905933,
+        -376.5651820410446
+    ],
+    "eiger_fw_version": "release-2022.1.2",
+    "element": "",
+    "flatfield_correction_applied": true,
+    "frame_count_time": 0.00344898,
+    "frame_period": 0.00344898,
+    "frame_time": 0.1,
+    "kappa_increment": 0.0,
+    "kappa_start": 0.0,
+    "nimages": 1,
+    "ntrigger": 1,
+    "number_of_excluded_pixels": 1253074,
+    "omega_increment": 0.1,
+    "omega_start": -90.0,
+    "phi_increment": 0.0,
+    "phi_start": 0.0,
+    "photon_energy": 12699.699839514962,
+    "pixel_mask_applied": true,
+    "roi_mode": "",
+    "sensor_material": "Si",
+    "sensor_thickness": 0.00045,
+    "software_version": "1.8.0",
+    "threshold_energy": 6349.849919757481,
+    "trigger_mode": "ints",
+    "two_theta_increment": 0.0,
+    "two_theta_start": 0.0,
+    "virtual_pixel_correction_applied": true,
+    "wavelength": 0.976276604959,
+    "x_pixel_size": 7.5e-05,
+    "x_pixels_in_detector": 4148,
+    "y_pixel_size": 7.5e-05,
+    "y_pixels_in_detector": 4362
+}

python/tests/input/06.json

@@ -0,0 +1,13 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-globalflatfield",
+    "type": "raw",
+    "header": {
+        "acqID": "test_0_0",
+        "shape": [
+            4148,
+            4362
+        ],
+        "type": "float32"
+    }
+}

python/tests/input/08.json

@@ -0,0 +1,13 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-globalmask",
+    "type": "raw",
+    "header": {
+        "acqID": "test_0_0",
+        "shape": [
+            4148,
+            4362
+        ],
+        "type": "uint32"
+    }
+}

python/tests/input/10.json

@@ -0,0 +1,13 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-globalcountrate",
+    "type": "raw",
+    "header": {
+        "acqID": "test_0_0",
+        "shape": [
+            2,
+            1152
+        ],
+        "type": "float32"
+    }
+}

python/tests/input/12.json

@@ -0,0 +1,21 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-imagedata",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "frame": 0,
+        "series": 40181,
+        "size": 1504987,
+        "start_time": 3964989369387280,
+        "stop_time": 3964989469387180,
+        "real_time": 99999900,
+        "shape": [
+            4148,
+            4362
+        ],
+        "type": "uint32",
+        "encoding": "bs32-lz4<",
+        "hash": "3aca01d1417576a9c8c26343a93b29a1"
+    }
+}

python/tests/input/13.json

@@ -0,0 +1,16 @@
+{
+    "acqID": "test_0_0",
+    "frame": 0,
+    "series": 40181,
+    "size": 1504987,
+    "start_time": 3964989369387280,
+    "stop_time": 3964989469387180,
+    "real_time": 99999900,
+    "shape": [
+        4148,
+        4362
+    ],
+    "type": "uint32",
+    "encoding": "bs32-lz4<",
+    "hash": "3aca01d1417576a9c8c26343a93b29a1"
+}

python/tests/input/14.json

@@ -0,0 +1,9 @@
+{
+    "plugin": "Acquisition",
+    "parameter": "writeframe",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "rank": 0
+    }
+}

python/tests/input/15.json

@@ -0,0 +1,7 @@
+{
+    "frame": 0,
+    "offset": 0,
+    "proc": 4,
+    "write_duration": 3550,
+    "flush_duration": 60
+}

python/tests/input/16.json

@@ -0,0 +1,9 @@
+{
+    "plugin": "Acquisition",
+    "parameter": "closefile",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "rank": 0
+    }
+}

python/tests/input/17.json

@@ -0,0 +1,4 @@
+{
+    "filePath": "/dls/i03/data/2024/cm37235-3/tmp/gary/0/test_0_0_000001.h5",
+    "close_duration": 263
+}

python/tests/input/18.json

@@ -0,0 +1,9 @@
+{
+    "plugin": "Acquisition",
+    "parameter": "stopacquisition",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "rank": 0
+    }
+}

python/tests/input/20.json

@@ -0,0 +1,9 @@
+{
+    "plugin": "eiger",
+    "parameter": "eiger-end",
+    "type": "string",
+    "header": {
+        "acqID": "test_0_0",
+        "series": 40181
+    }
+}

python/tests/test_eiger_meta_writer.py

@@ -1,9 +1,38 @@
+import json
+from pathlib import Path
+
+import h5py as h5
 from eiger_detector.data.eiger_meta_writer import EigerMetaWriter
 from odin_data.meta_writer.meta_writer import MetaWriterConfig
 
+HERE = Path(__file__).parent
+
+
+def test_file_write(tmp_path):
 
-def test_eiger_meta_writer():
     writer = EigerMetaWriter(
-        "writer", "/tmp", [], MetaWriterConfig(sensor_shape=(4362, 4148))
+        "test", tmp_path.as_posix(), [], MetaWriterConfig(sensor_shape=(4362, 4148))
     )
-    assert writer._name == "writer"
+
+    writer._processes_running = [True]
+    writer._endpoints = ["tcp://127.0.0.1:10008"]
+
+    stream_files = sorted((HERE / "input").iterdir())
+
+    # Every other file is header and then data
+    for header_path, data_path in zip(stream_files[0::2], stream_files[1::2]):
+        with header_path.open("r") as f:
+            header = json.load(f)
+            header["header"]["_endpoint"] = "tcp://127.0.0.1:10008"
+
+        if data_path.suffix == ".json":
+            with data_path.open("r") as f:
+                data = json.load(f)
+        else:
+            with data_path.open("rb") as f:
+                data = f.read()
+
+        writer.process_message(header, data)
+
+    with h5.File(tmp_path / "test_meta.h5") as f:
+        assert f["series"][0] == 40181