scdms_soudan_minimal_file.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from construct import *\n",
    "\n",
    "format_word = Struct(\n",
    "    \"daq_major\" / Byte,\n",
    "    \"daq_minor\" / Byte,\n",
    "    \"data_format_major\" / Byte,\n",
    "    \"data_format_minor\" / Byte,\n",
    ")\n",
    "\n",
    "two_word_file_header = Struct(\"endian_indicator\" / Int32ul, \"data_format\" / format_word)\n",
    "\n",
    "detector_hdr = Struct(\n",
    "    \"header_number\" / Int32ul,\n",
    "    \"config_record_len\" / Int32ul,\n",
    "    \"repeat_value\"\n",
    "    / Computed(\n",
    "        lambda this: (this.config_record_len // 72) + (this.config_record_len // 144)\n",
    "    ),\n",
    ")\n",
    "\n",
    "charge_config_header = Struct(\n",
    "    \"charge_config_len\" / Int32ul,\n",
    "    \"detector_code\" / Int32sl,\n",
    "    \"tower_number\" / Int32sl,\n",
    "    \"channel_post_amp\" / Int32sl,\n",
    "    \"channel_bias\" / Int32sl,\n",
    "    \"rtf_offset\" / Int32sl,\n",
    "    \"delta_t\" / Int32sl,\n",
    "    \"trigger_time\" / Int32sl,\n",
    "    \"trace_len\" / Int32sl,\n",
    ")\n",
    "\n",
    "phonon_config_header = Struct(\n",
    "    \"phonon_config_len\" / Int32ul,\n",
    "    \"detector_code\" / Int32sl,\n",
    "    \"tower_number\" / Int32sl,\n",
    "    \"post_amp_gain\" / Int32sl,\n",
    "    \"qet_bias\" / Int32sl,\n",
    "    \"squid_bias\" / Int32sl,\n",
    "    \"squid_lockpoint\" / Int32sl,\n",
    "    \"rtf_offset\" / Int32sl,\n",
    "    \"variable_gain\" / Int32sl,\n",
    "    \"delta_t\" / Int32sl,\n",
    "    \"trigger_time\" / Int32sl,\n",
    "    \"trace_len\" / Int32sl,\n",
    ")\n",
    "\n",
    "header_list = Struct(\n",
    "    \"header_number\" / Int32ul,\n",
    "    \"charge_config\"\n",
    "    / If(lambda this: this.header_number == 0x10002, charge_config_header),\n",
    "    \"phonon_config\"\n",
    "    / If(lambda this: this.header_number == 0x10001, phonon_config_header),\n",
    ")\n",
    "\n",
    "event_header = Struct(\n",
    "    \"event_header_word\" / Int32ul,\n",
    "    \"event_size\" / Int32ul,\n",
    "    \"event_identifier\" / Computed(lambda this: (this.event_header_word >> 16) & 0xFFFF),\n",
    "    # 0x0: Raw, 0x1: Processed, 0x2: Monte Carlo\n",
    "    \"event_class\" / Computed(lambda this: (this.event_header_word >> 8) & 0xF),\n",
    "    # 0x0: Per Trigger, 0x1: Occasional, 0x2: Begin File Series, 0x3: Begin File\n",
    "    # 0x4: End File, 0x5: End File Series, 0x6: Per Trigger w/ Detectors that Cross Threshold\n",
    "    \"event_category\" / Computed(lambda this: (this.event_header_word >> 12) & 0xF),\n",
    "    # 0x0: Wimp Search, 0x1: 60Co Calibration, 0x2: 60Co Low Energy Calibration,\n",
    "    # 0x3: Neutron Calibration, 0x4: Random Triggers, 0x5: Pulse Triggers\n",
    "    # 0x6: Test, 0x7: Data Monitering Event, 0x8: 137Cs Calibration\n",
    "    \"event_type\" / Computed(lambda this: (this.event_header_word & 0xFF)),\n",
    ")\n",
    "\n",
    "administrative_record = Struct(\n",
    "    \"admin_header\" / Int32ul,\n",
    "    \"admin_len\" / Int32ul,\n",
    "    \"series_number_1\" / Int32ul,\n",
    "    \"series_number_2\" / Int32ul,\n",
    "    \"event_number_in_series\" / Int32ul,\n",
    "    \"seconds_from_epoch\" / Int32ul,\n",
    "    # Epoch defined as Jan 1st 1904 for SUF (MAC Artifact)\n",
    "    # Epoch defined as Jan 1st 1970 for Soudan\n",
    "    \"time_from_last_event\" / Int32ul,\n",
    "    \"live_time_from_last_event\" / Int32ul,\n",
    ")\n",
    "\n",
    "trace_record = Struct(\n",
    "    \"trace_header\" / Int32ul,\n",
    "    \"trace_len\" / Int32ul,\n",
    "    \"trace_bookkeeping_header\" / Int32ul,\n",
    "    \"bookkeeping_len\" / Int32ul,\n",
    "    \"digitizer_base_address\" / Int32ul,\n",
    "    \"digitizer_channel\" / Int32ul,\n",
    "    \"detector_code\" / Int32ul,\n",
    "    \"timebase_header\" / Int32ul,\n",
    "    \"timebase_len\" / Int32ul,\n",
    "    \"t0_in_ns\" / Int32ul,\n",
    "    \"delta_t_ns\" / Int32ul,\n",
    "    \"num_of_points\" / Int32ul,\n",
    "    \"second_trace_header\" / Int32ul,\n",
    "    \"num_samples\" / Int32ul,\n",
    "    # Should be a power of two (1024, 2048, etc)\n",
    ")\n",
    "\n",
    "data_sample = Struct(\n",
    "    \"data_selection\" / Int32ul,\n",
    "    \"sample_a\" / Computed(lambda this: (this.data_selection >> 16) & 0xFFFF),\n",
    "    \"sample_b\" / Computed(lambda this: (this.data_selection & 0xFFFF)),\n",
    ")\n",
    "\n",
    "\n",
    "trace_data = Struct(\n",
    "    \"trace_rcrds\" / trace_record,\n",
    "    \"sample_data\" / Array(this.trace_rcrds.num_samples // 2, data_sample),\n",
    ")\n",
    "\n",
    "soudan_history_buffer = Struct(\n",
    "    \"history_buffer_header\" / Int32ul,\n",
    "    \"history_buffer_len\" / Int32ul,\n",
    "    \"num_time_nvt\" / Int32ul,\n",
    "    \"time_nvt\" / Array(this.num_time_nvt, Int32ul),\n",
    "    \"num_veto_mask_words\" / Int32ul,\n",
    "    \"time_n_minus_veto_mask\"\n",
    "    / Array(this.num_time_nvt * this.num_veto_mask_words, Int32ul),\n",
    "    \"num_trigger_times\" / Int32ul,\n",
    "    \"trigger_times\" / Array(this.num_trigger_times, Int32ul),\n",
    "    \"num_trigger_mask_words\" / Int32ul,\n",
    "    \"trig_times_minus_trig_mask\"\n",
    "    / Array(this.num_trigger_times * this.num_trigger_mask_words, Int32ul),\n",
    ")\n",
    "\n",
    "trigger_record = Struct(\n",
    "    \"trigger_header\" / Int32ul,\n",
    "    \"trigger_len\" / Int32ul,\n",
    "    \"trigger_time\" / Int32ul,\n",
    "    \"individual_trigger_masks\" / Array(6, Int32ul),\n",
    ")\n",
    "\n",
    "tlb_trigger_mask_record = Struct(\n",
    "    \"tlb_mask_header\" / Int32ul, \"tlb_len\" / Int32ul, \"tower_mask\" / Array(6, Int32ul)\n",
    ")\n",
    "\n",
    "gps_data = Struct(\n",
    "    \"tlb_mask_header\" / Int32ul,\n",
    "    \"length\" / Int32ul,\n",
    "    \"gps_year_day\" / If(this.length > 0, Int32ul),\n",
    "    \"gps_status_hour_minute_second\" / If(this.length > 0, Int32ul),\n",
    "    \"gps_microsecs_from_gps_second\" / If(this.length > 0, Int32ul),\n",
    ")\n",
    "\n",
    "detector_trigger_threshold_data = Struct(\n",
    "    \"threshold_header\" / Int32ul,\n",
    "    \"len_to_next_header\" / Int32ul,\n",
    "    \"minimum_voltage_level\" / Int32ul,\n",
    "    \"maximum_voltage_level\" / Int32ul,\n",
    "    \"dynamic_range\" / Int32ul,\n",
    "    \"tower_number\" / Int32ul,\n",
    "    \"detector_codes\" / Array(6, Int32ul),\n",
    "    \"operations_codes\" / Array(9, Int32ul),\n",
    "    \"adc_values\" / Array(54, Int32ul),\n",
    ")\n",
    "\n",
    "detector_trigger_rates = Struct(\n",
    "    \"detector_trigger_header\" / Int32ul,\n",
    "    \"len_to_next_header\" / Int32ul,\n",
    "    \"clocking_interval\" / Int32ul,\n",
    "    \"tower_number\" / Int32ul,\n",
    "    \"detector_codes\" / Array(6, Int32ul),\n",
    "    \"j_codes\" / Array(5, Int32ul),\n",
    "    \"counter_values\" / Array(30, Int32ul),\n",
    ")\n",
    "\n",
    "veto_trigger_rates = Struct(\n",
    "    \"veto_trigger_header\" / Int32ul,\n",
    "    \"len_to_next_header\" / Int32ul,\n",
    "    \"clocking_interval\" / Int32ul,\n",
    "    \"num_entries\" / Int32ul,\n",
    "    \"detector_code\" / Array(this.num_entries, Int32ul),\n",
    "    \"counter_value_det_code\" / Array(this.num_entries, Int32ul),\n",
    ")\n",
    "\n",
    "\n",
    "logical_records = Struct(\n",
    "    \"event_hdr\" / Peek(Int32ul),  # Peek to check first\n",
    "    \"next_section\"\n",
    "    / Struct(\n",
    "        \"next_header\" / Peek(Int32ul),  # Peek without consuming\n",
    "        \"section\"\n",
    "        / Switch(\n",
    "            lambda this: (\n",
    "                this.next_header\n",
    "                if ((this.next_header >> 16) != 0xA980)\n",
    "                else 0xA980  # Use 0xA980 as identifier for event_header\n",
    "            ),\n",
    "            {\n",
    "                0xA980: event_header,\n",
    "                0x00000002: administrative_record,\n",
    "                0x00000011: trace_data,\n",
    "                0x00000021: soudan_history_buffer,\n",
    "                0x00000060: gps_data,\n",
    "                0x00000080: trigger_record,\n",
    "                0x00000081: tlb_trigger_mask_record,\n",
    "                0x00000022: detector_trigger_rates,\n",
    "                0x00000031: veto_trigger_rates,\n",
    "            },\n",
    "        ),\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soudan = Struct(\n",
    "    \"file_hdr\" / two_word_file_header,\n",
    "    \"detector_hdr\" / detector_hdr,\n",
    "    \"hdrs\" / Array(this._root.detector_hdr.repeat_value, header_list),\n",
    "    \"logical_rcrds\" / GreedyRange(logical_records),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_hdr = {\n",
    "    \"endian_indicator\": 1,\n",
    "    \"data_format\": {\n",
    "        \"daq_major\": 1,\n",
    "        \"daq_minor\": 2,\n",
    "        \"data_format_major\": 3,\n",
    "        \"data_format_minor\": 4,\n",
    "    },\n",
    "}\n",
    "soudan.file_hdr.build(test_file_hdr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_detector_hdr = {\n",
    "    \"header_number\": 0x10002,\n",
    "    \"config_record_len\": 72 * 2,\n",
    "}\n",
    "soudan.detector_hdr.build(test_detector_hdr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repeat_value = soudan.detector_hdr.parse(_).repeat_value\n",
    "repeat_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_hdrs = []\n",
    "for i in range(repeat_value - 1):\n",
    "    test_hdrs.append(\n",
    "        {\n",
    "            \"header_number\": 0x10002,\n",
    "            \"charge_config\": {\n",
    "                \"charge_config_len\": 72,\n",
    "                \"detector_code\": 1,\n",
    "                \"tower_number\": 1,\n",
    "                \"channel_post_amp\": 1,\n",
    "                \"channel_bias\": 1,\n",
    "                \"rtf_offset\": 1,\n",
    "                \"delta_t\": 1,\n",
    "                \"trigger_time\": 1,\n",
    "                \"trace_len\": 1,\n",
    "            },\n",
    "            \"phonon_config\": {},\n",
    "        }\n",
    "    )\n",
    "test_hdrs.append(\n",
    "    {\n",
    "        \"header_number\": 0x10001,\n",
    "        \"phonon_config\": {\n",
    "            \"phonon_config_len\": 72,\n",
    "            \"detector_code\": 1,\n",
    "            \"tower_number\": 1,\n",
    "            \"post_amp_gain\": 1,\n",
    "            \"qet_bias\": 1,\n",
    "            \"squid_bias\": 1,\n",
    "            \"squid_lockpoint\": 1,\n",
    "            \"rtf_offset\": 1,\n",
    "            \"variable_gain\": 1,\n",
    "            \"delta_t\": 1,\n",
    "            \"trigger_time\": 1,\n",
    "            \"trace_len\": 1,\n",
    "        },\n",
    "        \"charge_config\": {},\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soudan_test_hdrs = Struct(\n",
    "    \"hdrs\" / Array(repeat_value, header_list),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soudan_test_hdrs.build({\"hdrs\": test_hdrs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soudan_test_hdrs.parse(_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_logical_records = [\n",
    "    {\n",
    "        # Event Header Record\n",
    "        \"event_hdr\": 0xA9800000,\n",
    "        \"next_section\": {\n",
    "            \"next_header\": 0xA9800000,\n",
    "            \"section\": {\n",
    "                \"event_header_word\": 0xA9800000,\n",
    "                \"event_size\": 100,\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # Admin Header Record\n",
    "        \"event_hdr\": 0x00000002,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000002,\n",
    "            \"section\": {\n",
    "                # Admin Section\n",
    "                \"admin_header\": 0x00000002,\n",
    "                \"admin_len\": 50,\n",
    "                \"series_number_1\": 1,\n",
    "                \"series_number_2\": 2,\n",
    "                \"event_number_in_series\": 3,\n",
    "                \"seconds_from_epoch\": 1000,\n",
    "                \"time_from_last_event\": 10,\n",
    "                \"live_time_from_last_event\": 5,\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # Trace Header Record\n",
    "        \"event_hdr\": 0x00000011,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000011,\n",
    "            \"section\": {\n",
    "                \"trace_rcrds\": {\n",
    "                    # Trace Section\n",
    "                    \"trace_header\": 0x00000011,\n",
    "                    \"trace_len\": 200,\n",
    "                    \"trace_bookkeeping_header\": 0x00000011,\n",
    "                    \"bookkeeping_len\": 20,\n",
    "                    \"digitizer_base_address\": 0x1000,\n",
    "                    \"digitizer_channel\": 1,\n",
    "                    \"detector_code\": 1,\n",
    "                    \"timebase_header\": 0x00000011,\n",
    "                    \"timebase_len\": 10,\n",
    "                    \"t0_in_ns\": 100,\n",
    "                    \"delta_t_ns\": 10,\n",
    "                    \"num_of_points\": 1000,\n",
    "                    \"second_trace_header\": 0x00000011,\n",
    "                    \"num_samples\": 4,  # This should agree with the number of elements in sample_data\n",
    "                },\n",
    "                \"sample_data\": [\n",
    "                    {\"data_selection\": 0x00010001, \"sample_a\": 1, \"sample_b\": 1},\n",
    "                    {\"data_selection\": 0x00020002, \"sample_a\": 2, \"sample_b\": 2},\n",
    "                ],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # History Buffer Header Record\n",
    "        \"event_hdr\": 0x00000021,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000021,\n",
    "            \"section\": {\n",
    "                # History Buffer Section\n",
    "                \"history_buffer_header\": 0x00000021,\n",
    "                \"history_buffer_len\": 100,\n",
    "                \"num_time_nvt\": 2,\n",
    "                \"time_nvt\": [100, 200],\n",
    "                \"num_veto_mask_words\": 2,\n",
    "                \"time_n_minus_veto_mask\": [300, 400, 500, 600],\n",
    "                \"num_trigger_times\": 2,\n",
    "                \"trigger_times\": [500, 600],\n",
    "                \"num_trigger_mask_words\": 2,\n",
    "                \"trig_times_minus_trig_mask\": [700, 800, 900, 1000],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # TLB Mask Header Record\n",
    "        \"event_hdr\": 0x00000060,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000060,\n",
    "            \"section\": {\n",
    "                # TLB Mask Section\n",
    "                \"tlb_mask_header\": 0x00000060,\n",
    "                \"length\": 10,\n",
    "                \"gps_year_day\": 2021365,\n",
    "                \"gps_status_hour_minute_second\": 123456,\n",
    "                \"gps_microsecs_from_gps_second\": 789,\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # Trigger Header Record\n",
    "        \"event_hdr\": 0x00000080,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000080,\n",
    "            \"section\": {\n",
    "                # Trigger Section\n",
    "                \"trigger_header\": 0x00000080,\n",
    "                \"trigger_len\": 50,\n",
    "                \"trigger_time\": 1000,\n",
    "                \"individual_trigger_masks\": [1, 2, 3, 4, 5, 6],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # TLB Mask Header Record\n",
    "        \"event_hdr\": 0x00000081,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000081,\n",
    "            \"section\": {\n",
    "                # TLB Mask Section\n",
    "                \"tlb_mask_header\": 0x00000081,\n",
    "                \"tlb_len\": 50,\n",
    "                \"tower_mask\": [1, 2, 3, 4, 5, 6],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # Detector Trigger Header Record\n",
    "        \"event_hdr\": 0x00000022,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000022,\n",
    "            \"section\": {\n",
    "                # Detector Trigger Section\n",
    "                \"detector_trigger_header\": 0x00000022,\n",
    "                \"len_to_next_header\": 50,\n",
    "                \"clocking_interval\": 100,\n",
    "                \"tower_number\": 1,\n",
    "                \"detector_codes\": [1, 2, 3, 4, 5, 6],\n",
    "                \"j_codes\": [1, 2, 3, 4, 5],\n",
    "                \"counter_values\": [\n",
    "                    1,\n",
    "                    2,\n",
    "                    3,\n",
    "                    4,\n",
    "                    5,\n",
    "                    6,\n",
    "                    7,\n",
    "                    8,\n",
    "                    9,\n",
    "                    10,\n",
    "                    11,\n",
    "                    12,\n",
    "                    13,\n",
    "                    14,\n",
    "                    15,\n",
    "                    16,\n",
    "                    17,\n",
    "                    18,\n",
    "                    19,\n",
    "                    20,\n",
    "                    21,\n",
    "                    22,\n",
    "                    23,\n",
    "                    24,\n",
    "                    25,\n",
    "                    26,\n",
    "                    27,\n",
    "                    28,\n",
    "                    29,\n",
    "                    30,\n",
    "                ],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        # Veto Trigger Header Record\n",
    "        \"event_hdr\": 0x00000031,\n",
    "        \"next_section\": {\n",
    "            # Next Section Header\n",
    "            \"next_header\": 0x00000031,\n",
    "            \"section\": {\n",
    "                # Veto Trigger Section\n",
    "                \"veto_trigger_header\": 0x00000031,\n",
    "                \"len_to_next_header\": 50,\n",
    "                \"clocking_interval\": 100,\n",
    "                \"num_entries\": 2,\n",
    "                \"detector_code\": [1, 2],\n",
    "                \"counter_value_det_code\": [3, 4],\n",
    "            },\n",
    "        },\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the logical records\n",
    "built_logical_records = soudan.logical_rcrds.build(test_logical_records)\n",
    "\n",
    "# Parse the built logical records\n",
    "parsed_logical_records = soudan.logical_rcrds.parse(built_logical_records)\n",
    "\n",
    "parsed_logical_records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "complete_test_file = {\n",
    "    \"file_hdr\": test_file_hdr,\n",
    "    \"detector_hdr\": test_detector_hdr,\n",
    "    \"hdrs\": test_hdrs,\n",
    "    \"logical_rcrds\": test_logical_records,}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the complete test file\n",
    "built_complete_test_file = soudan.build(complete_test_file)\n",
    "\n",
    "# Write the built file to a binary file\n",
    "with open('minimal_sample.soudan', 'wb') as f:\n",
    "    f.write(built_complete_test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -lah minimal_sample.soudan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cdms",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}