Merge pull request #2903 from Graenni/implement_pyPublicDecompWT_hrit…

…_base

Implement py public decomp wt hrit base

AUTHORS.md

@@ -76,6 +76,7 @@ The following people have made contributions to this project:
 - Marco Sassi - meteoswiss
 - [Stefan Scheiblauer (StefanSnippetCoder)](https:/StefanSnippetCoder)
 - [Ronald Scheirer](https:/)
+- [Michael Schmutz (Graenni)](https:/Graenni) - Meteotest AG
 - [Hauke Schulz (observingClouds)](https:/observingClouds)
 - [Jakub Seidl (seidlj)](https:/seidlj)
 - [Eysteinn Sigurðsson (eysteinn)](https:/eysteinn)

continuous_integration/environment.yaml

@@ -64,3 +64,4 @@ dependencies:
  - trollimage>=1.24
  - pyspectral
  - pyorbital
+ - pyPublicDecompWT

pyproject.toml

@@ -55,7 +55,7 @@ msi_safe = ["rioxarray", "bottleneck", "python-geotiepoints", "defusedxml"]
 nc_nwcsaf_msg = ["netCDF4 >= 1.1.8"]
 sar_c = ["python-geotiepoints >= 1.1.7", "rasterio", "rioxarray", "defusedxml"]
 abi_l1b = ["h5netcdf"]
-seviri_l1b_hrit = ["pyorbital >= 1.3.1"]
+seviri_l1b_hrit = ["pyorbital >= 1.3.1", "pyPublicDecompWT"]
 seviri_l1b_native = ["pyorbital >= 1.3.1"]
 seviri_l1b_nc = ["pyorbital >= 1.3.1", "netCDF4 >= 1.1.8"]
 seviri_l2_bufr = ["eccodes"]

satpy/readers/hrit_base.py

@@ -21,20 +21,13 @@
 This module is the base module for all HRIT-based formats. Here, you will find
 the common building blocks for hrit reading.
 
-One of the features here is the on-the-fly decompression of hrit files. It needs
-a path to the xRITDecompress binary to be provided through the environment
-variable called XRIT_DECOMPRESS_PATH. When compressed hrit files are then
-encountered (files finishing with `.C_`), they are decompressed to the system's
-temporary directory for reading.
-
+One of the features here is the on-the-fly decompression of hrit files when
+compressed hrit files are encountered (files finishing with `.C_`).
 """
 
 import datetime as dt
 import logging
 import os
-from contextlib import contextmanager, nullcontext
-from io import BytesIO
-from subprocess import PIPE, Popen # nosec B404
 
 import dask
 import dask.array as da
@@ -43,7 +36,6 @@
 from pyresample import geometry
 
 import satpy.readers.utils as utils
-from satpy import config
 from satpy.readers import FSFile
 from satpy.readers.eum_base import time_cds_short
 from satpy.readers.file_handlers import BaseFileHandler
@@ -96,61 +88,16 @@
  }
 
 
-def get_xritdecompress_cmd():
- """Find a valid binary for the xRITDecompress command."""
- cmd = os.environ.get("XRIT_DECOMPRESS_PATH", None)
- if not cmd:
- raise IOError("XRIT_DECOMPRESS_PATH is not defined (complete path to xRITDecompress)")
-
- question = ("Did you set the environment variable XRIT_DECOMPRESS_PATH correctly?")
- if not os.path.exists(cmd):
- raise IOError(str(cmd) + " does not exist!\n" + question)
- elif os.path.isdir(cmd):
- raise IOError(str(cmd) + " is a directory!\n" + question)
-
- return cmd
-
-
-def get_xritdecompress_outfile(stdout):
- """Analyse the output of the xRITDecompress command call and return the file."""
- outfile = b""
- for line in stdout:
- try:
- k, v = [x.strip() for x in line.split(b":", 1)]
- except ValueError:
- break
- if k == b"Decompressed file":
- outfile = v
- break
-
- return outfile
-
-
-def decompress(infile, outdir="."):
- """Decompress an XRIT data file and return the path to the decompressed file.
+def decompress(infile):
+ """Decompress an XRIT data file and return the decompressed buffer."""
+ from pyPublicDecompWT import xRITDecompress
 
- It expect to find Eumetsat's xRITDecompress through the environment variable
- XRIT_DECOMPRESS_PATH.
- """
- cmd = get_xritdecompress_cmd()
- infile = os.path.abspath(infile)
- cwd = os.getcwd()
- os.chdir(outdir)
+ # decompress in-memory
+ with open(infile, mode="rb") as fh:
+ xrit = xRITDecompress()
+ xrit.decompress(fh.read())
 
- p = Popen([cmd, infile], stdout=PIPE) # nosec B603
- stdout = BytesIO(p.communicate()[0])
- status = p.returncode
- os.chdir(cwd)
-
- if status != 0:
- raise IOError("xrit_decompress '%s', failed, status=%d" % (infile, status))
-
- outfile = get_xritdecompress_outfile(stdout)
-
- if not outfile:
- raise IOError("xrit_decompress '%s', failed, no output file is generated" % infile)
-
- return os.path.join(outdir, outfile.decode("utf-8"))
+ return xrit.data()
 
 
 def get_header_id(fp):
@@ -343,18 +290,6 @@ def _read_data(filename, mda):
  return HRITSegment(filename, mda).read_data()
 
 
-@contextmanager
-def decompressed(filename):
- """Decompress context manager."""
- try:
- new_filename = decompress(filename, config["tmp_dir"])
- except IOError as err:
- logger.error("Unpacking failed: %s", str(err))
- raise
- yield new_filename
- os.remove(new_filename)
-
-
 class HRITSegment:
  """An HRIT segment with data."""
 
@@ -389,11 +324,21 @@ def _read_data_from_disk(self):
  # For reading the image data, unzip_context is faster than generic_open
  dtype, shape = self._get_input_info()
  with utils.unzip_context(self.filename) as fn:
- with decompressed(fn) if self.compressed else nullcontext(fn) as filename:
- return np.fromfile(filename,
- offset=self.offset,
- dtype=dtype,
- count=np.prod(shape))
+
+ if self.compressed:
+ return np.frombuffer(
+ decompress(fn),
+ offset=self.offset,
+ dtype=dtype,
+ count=np.prod(shape)
+ )
+ else:
+ return np.fromfile(
+ fn,
+ offset=self.offset,
+ dtype=dtype,
+ count=np.prod(shape)
+ )
 
  def _read_file_like(self):
  # filename is likely to be a file-like object, already in memory

satpy/tests/reader_tests/test_hrit_base.py

@@ -21,75 +21,20 @@
 import datetime as dt
 import gzip
 import os
-import unittest
-from tempfile import NamedTemporaryFile, gettempdir
 from unittest import mock
 
 import numpy as np
 import pytest
 
 from satpy.readers import FSFile
-from satpy.readers.hrit_base import HRITFileHandler, decompress, get_xritdecompress_cmd, get_xritdecompress_outfile
+from satpy.readers.hrit_base import HRITFileHandler
 from satpy.tests.utils import RANDOM_GEN
 
 # NOTE:
 # The following fixtures are not defined in this file, but are used and injected by Pytest:
 # - tmp_path
 
 
-class TestHRITDecompress(unittest.TestCase):
- """Test the on-the-fly decompression."""
-
- def test_xrit_cmd(self):
- """Test running the xrit decompress command."""
- old_env = os.environ.get("XRIT_DECOMPRESS_PATH", None)
-
- os.environ["XRIT_DECOMPRESS_PATH"] = "/path/to/my/bin"
- with pytest.raises(IOError, match=".* does not exist!"):
- get_xritdecompress_cmd()
-
- os.environ["XRIT_DECOMPRESS_PATH"] = gettempdir()
- with pytest.raises(IOError, match=".* is a directory!.*"):
- get_xritdecompress_cmd()
-
- with NamedTemporaryFile() as fd:
- os.environ["XRIT_DECOMPRESS_PATH"] = fd.name
- fname = fd.name
- res = get_xritdecompress_cmd()
-
- if old_env is not None:
- os.environ["XRIT_DECOMPRESS_PATH"] = old_env
- else:
- os.environ.pop("XRIT_DECOMPRESS_PATH")
-
- assert fname == res
-
- def test_xrit_outfile(self):
- """Test the right decompression filename is used."""
- stdout = [b"Decompressed file: bla.__\n"]
- outfile = get_xritdecompress_outfile(stdout)
- assert outfile == b"bla.__"
-
- @mock.patch("satpy.readers.hrit_base.Popen")
- def test_decompress(self, popen):
- """Test decompression works."""
- popen.return_value.returncode = 0
- popen.return_value.communicate.return_value = [b"Decompressed file: bla.__\n"]
-
- old_env = os.environ.get("XRIT_DECOMPRESS_PATH", None)
-
- with NamedTemporaryFile() as fd:
- os.environ["XRIT_DECOMPRESS_PATH"] = fd.name
- res = decompress("bla.C_")
-
- if old_env is not None:
- os.environ["XRIT_DECOMPRESS_PATH"] = old_env
- else:
- os.environ.pop("XRIT_DECOMPRESS_PATH")
-
- assert res == os.path.join(".", "bla.__")
-
-
 # From a compressed msg hrit file.
 # uncompressed data field length 17223680
 # compressed data field length 1578312
@@ -142,18 +87,25 @@ def stub_hrit_file(tmp_path):
 
 def create_stub_hrit(filename, open_fun=open, meta=mda):
  """Create a stub hrit file."""
+ stub_hrit_data = create_stub_hrit_data(meta)
+
+ with open_fun(filename, mode="wb") as fd:
+ fd.write(stub_hrit_data)
+ return filename
+
+def create_stub_hrit_data(meta):
+ """Create the data for the stub hrit."""
  nbits = meta["number_of_bits_per_pixel"]
  lines = meta["number_of_lines"]
  cols = meta["number_of_columns"]
  total_bits = lines * cols * nbits
  arr = RANDOM_GEN.integers(0, 256,
  size=int(total_bits / 8),
  dtype=np.uint8)
- with open_fun(filename, mode="wb") as fd:
- fd.write(b" " * meta["total_header_length"])
- bytes_data = arr.tobytes()
- fd.write(bytes_data)
- return filename
+ header_data = b" " * meta["total_header_length"]
+ bytes_data = arr.tobytes()
+ stub_hrit_data = header_data + bytes_data
+ return stub_hrit_data
 
 
 @pytest.fixture
@@ -276,10 +228,10 @@ def test_start_end_time(self):
  assert self.reader.end_time == self.reader.observation_end_time
 
 
-def fake_decompress(infile, outdir="."):
+def fake_decompress(filename):
  """Fake decompression."""
- filename = os.fspath(infile)[:-3]
- return create_stub_hrit(filename)
+ del filename
+ return create_stub_hrit_data(mda)
 
 
 class TestHRITFileHandlerCompressed: