Support HDF5 settings in pre-processed build_raw too.

src/daq2lh5/buffer_processor/buffer_processor.py

@@ -48,10 +48,17 @@ def buffer_processor(rb: RawBuffer) -> Table:
  ``"dtype_conv": {"lgdo": "dtype" [, ...]}`` `(dict)`
  Casts `lgdo` to the requested data type.
 
- ``"compression": { "lgdo": "codec_name" [, ...]}`` `(dict)`
+ ``"compression": {"lgdo": "codec_name" [, ...]}`` `(dict)`
  Updates the `compression` attribute of `lgdo` to `codec_name`. The
  attribute sets the compression algorithm applied by
  :func:`~.lgdo.lh5_store.LH5Store.read_object` before writing `lgdo` to
+ disk. Can be used to apply custom waveform compression algorithms from
+ :mod:`lgdo.compression`.
+
+ ``"hdf5_settings": {"lgdo": { <HDF5 settings> }}`` `(dict)`
+ Updates the `hdf5_settings` attribute of `lgdo`. The attribute sets the
+ HDF5 dataset options applied by
+ :func:`~.lgdo.lh5_store.LH5Store.read_object` before writing `lgdo` to
  disk.
 
  Parameters
@@ -102,7 +109,9 @@ def buffer_processor(rb: RawBuffer) -> Table:
  ,}
  "compression": {
  "windowed_waveform/values": RadwareSigcompress(codec_shift=-32768),
- "presummed_waveform/values": ULEB128ZigZagDiff(),
+ }
+ "hdf5_settings": {
+ "presummed_waveform/values": {"shuffle": True, "compression": "lzf"},
  }
  }
  },
@@ -143,7 +152,7 @@ def buffer_processor(rb: RawBuffer) -> Table:
  if "drop" in rb.proc_spec.keys():
  process_drop(rb, tmp_table)
 
- # at last, assign compression attributes
+ # assign compression attributes
  if "compression" in rb.proc_spec.keys():
  for name, codec in rb.proc_spec["compression"].items():
  ptr = tmp_table
@@ -154,6 +163,15 @@ def buffer_processor(rb: RawBuffer) -> Table:
  codec if isinstance(codec, WaveformCodec) else str2wfcodec(codec)
  )
 
+ # and HDF5 settings
+ if "hdf5_settings" in rb.proc_spec.keys():
+ for name, settings in rb.proc_spec["hdf5_settings"].items():
+ ptr = tmp_table
+ for word in name.split("/"):
+ ptr = ptr[word]
+
+ ptr.attrs["hdf5_settings"] = settings
+
  return tmp_table
 
 

tests/buffer_processor/test_buffer_processor.py

@@ -11,7 +11,7 @@
 from daq2lh5.build_raw import build_raw
 from daq2lh5.fc.fc_event_decoder import fc_decoded_values
 
-# skip compression in build_raw
+# skip waveform compression in build_raw
 fc_decoded_values["waveform"].pop("compression", None)
 
 config_dir = Path(__file__).parent / "test_buffer_processor_configs"
@@ -1409,6 +1409,9 @@ def test_buffer_processor_compression_settings(lgnd_test_data, tmptestdir):
  ),
  "presummed_waveform/values": ULEB128ZigZagDiff(),
  },
+ "hdf5_settings": {
+ "presummed_waveform/values": {"shuffle": True},
+ },
  },
  }
  }

tests/buffer_processor/test_lh5_buffer_processor.py

@@ -3,6 +3,7 @@
 import sys
 from pathlib import Path
 
+import h5py
 import lgdo
 import numpy as np
 from dspeed import build_processing_chain as bpc
@@ -11,7 +12,7 @@
 from daq2lh5.build_raw import build_raw
 from daq2lh5.fc.fc_event_decoder import fc_decoded_values
 
-# skip compression in build_raw
+# skip waveform compression in build_raw
 fc_decoded_values["waveform"].pop("compression", None)
 
 config_dir = Path(__file__).parent / "test_buffer_processor_configs"
@@ -308,7 +309,7 @@ def test_lh5_buffer_processor_separate_name_tables(lgnd_test_data):
  # Set up I/O files, including config
  daq_file = lgnd_test_data.get_path("fcio/L200-comm-20211130-phy-spms.fcio")
  raw_file = daq_file.replace(
- "L200-comm-20211130-phy-spms.fcio", " L200-comm-fake-geds-and-spms.lh5"
+ "L200-comm-20211130-phy-spms.fcio", "L200-comm-fake-geds-and-spms.lh5"
  )
  processed_file = raw_file.replace(
  "L200-comm-fake-geds-and-spms.lh5", "L200-comm-fake-geds-and-spms_proc.lh5"
@@ -1089,3 +1090,60 @@ def test_buffer_processor_all_pass(lgnd_test_data):
  proc_df = proc[obj].get_dataframe([str(sub_obj)])
 
  assert raw_df.equals(proc_df)
+
+
+def test_lh5_buffer_processor_hdf5_settings(lgnd_test_data):
+ # Set up I/O files, including config
+ daq_file = lgnd_test_data.get_path("fcio/L200-comm-20211130-phy-spms.fcio")
+ raw_file = daq_file.replace(
+ "L200-comm-20211130-phy-spms.fcio", "L200-comm-20211130-phy-spms.lh5"
+ )
+ processed_file = raw_file.replace(
+ "L200-comm-20211130-phy-spms.lh5", "L200-comm-20211130-phy-spms_proc.lh5"
+ )
+
+ out_spec = {
+ "FCEventDecoder": {
+ "ch{key}": {
+ "key_list": [[0, 6]],
+ "out_stream": processed_file + ":{name}",
+ "out_name": "raw",
+ "proc_spec": {
+ "window": ["waveform", 1000, -1000, "windowed_waveform"],
+ "dsp_config": {
+ "outputs": ["presum_2rate", "presummed_waveform"],
+ "processors": {
+ "presum_rate, presummed_waveform": {
+ "function": "presum",
+ "module": "dspeed.processors",
+ "args": [
+ "waveform",
+ 0,
+ "presum_rate",
+ "presummed_waveform(shape=len(waveform)/16, period=waveform.period*16, offset=waveform.offset)",
+ ],
+ "unit": "ADC",
+ }
+ },
+ },
+ "drop": ["waveform"],
+ "dtype_conv": {
+ "presummed_waveform/values": "uint32",
+ },
+ "hdf5_settings": {
+ "presummed_waveform/values": {
+ "compression": "lzf",
+ "shuffle": False,
+ },
+ },
+ },
+ }
+ }
+ }
+
+ # Build the raw file
+ build_raw(in_stream=daq_file, out_spec=out_spec, overwrite=True)
+
+ with h5py.File(processed_file) as f:
+ assert f["ch0"]["raw"]["presummed_waveform"]["values"].compression == "lzf"
+ assert f["ch0"]["raw"]["presummed_waveform"]["values"].shuffle is False