Merge pull request #1237 from sfinkens/pygac-metadata

More flexible way of passing avhrr_l1b_gaclac reader kwargs to pygac

satpy/readers/avhrr_l1b_gaclac.py

@@ -56,8 +56,7 @@ class GACLACFile(BaseFileHandler):
 
  def __init__(self, filename, filename_info, filetype_info,
  start_line=None, end_line=None, strip_invalid_coords=True,
- interpolate_coords=True, adjust_clock_drift=True,
- tle_dir=None, tle_name=None, tle_thresh=7):
+ interpolate_coords=True, **reader_kwargs):
  """Init the file handler.
 
  Args:
@@ -67,12 +66,8 @@ def __init__(self, filename, filename_info, filetype_info,
  the beginning/end of the orbit
  interpolate_coords: Interpolate coordinates from every eighth pixel
  to all pixels.
- adjust_clock_drift: Adjust the geolocation to compensate for the
- clock error (POD satellites only).
- tle_dir: Directory holding Two-Line-Element (TLE) files
- tle_name: Filename pattern of TLE files.
- tle_thresh: Maximum number of days between observation and nearest
- TLE
+ reader_kwargs: More keyword arguments to be passed to pygac.Reader.
+ See the pygac documentation for available options.
 
  """
  super(GACLACFile, self).__init__(
@@ -82,18 +77,13 @@ def __init__(self, filename, filename_info, filetype_info,
  self.end_line = end_line
  self.strip_invalid_coords = strip_invalid_coords
  self.interpolate_coords = interpolate_coords
- self.adjust_clock_drift = adjust_clock_drift
- self.tle_dir = tle_dir
- self.tle_name = tle_name
- self.tle_thresh = tle_thresh
+ self.reader_kwargs = reader_kwargs
  self.creation_site = filename_info.get('creation_site')
  self.reader = None
  self.calib_channels = None
  self.counts = None
  self.angles = None
  self.qual_flags = None
- self.midnight_scanline = None
- self.missing_scanlines = None
  self.first_valid_lat = None
  self.last_valid_lat = None
  self._start_time = filename_info['start_time']
@@ -126,20 +116,20 @@ def __init__(self, filename, filename_info, filetype_info,
  self.sensor = 'avhrr'
  self.filename_info = filename_info
 
- def get_dataset(self, key, info):
- """Get the dataset."""
+ def read_raw_data(self):
+ """Create a pygac reader and read raw data from the file."""
  if self.reader is None:
  self.reader = self.reader_class(
  interpolate_coords=self.interpolate_coords,
- adjust_clock_drift=self.adjust_clock_drift,
- tle_dir=self.tle_dir,
- tle_name=self.tle_name,
- tle_thresh=self.tle_thresh,
- creation_site=self.creation_site)
+ creation_site=self.creation_site,
+ **self.reader_kwargs)
  self.reader.read(self.filename)
- if np.all(self.reader.mask):
- raise ValueError('All data is masked out')
+  if np.all(self.reader.mask):
+  raise ValueError('All data is masked out')
 
+ def get_dataset(self, key, info):
+ """Get the dataset."""
+ self.read_raw_data()
  if key.name in ['latitude', 'longitude']:
  # Lats/lons are buffered by the reader
  if key.name == 'latitude':
@@ -179,8 +169,6 @@ def get_dataset(self, key, info):
  self._end_time = times[-1].astype(datetime)
 
  # Select user-defined scanlines and/or strip invalid coordinates
- self.midnight_scanline = self.reader.meta_data['midnight_scanline']
- self.missing_scanlines = self.reader.meta_data['missing_scanlines']
  if (self.start_line is not None or self.end_line is not None
  or self.strip_invalid_coords):
  data, times = self.slice(data=data, times=times)
@@ -205,7 +193,7 @@ def get_dataset(self, key, info):
  def slice(self, data, times):
  """Select user-defined scanlines and/or strip invalid coordinates.
 
- Furthermore, update scanline timestamps and auxiliary information.
+ Furthermore, update scanline timestamps.
 
  Args:
  data: Data to be sliced
@@ -214,22 +202,17 @@ def slice(self, data, times):
  Sliced data and timestamps
 
  """
- # Slice data, update midnight scanline & list of missing scanlines
- sliced, self.midnight_scanline, miss_lines = self._slice(data)
- self.missing_scanlines = miss_lines.astype(int)
-
- # Slice timestamps, update start/end time
- times, _, _ = self._slice(times)
+ sliced = self._slice(data)
+ times = self._slice(times)
  self._start_time = times[0].astype(datetime)
  self._end_time = times[-1].astype(datetime)
-
  return sliced, times
 
  def _slice(self, data):
  """Select user-defined scanlines and/or strip invalid coordinates.
 
  Returns:
- Sliced data, updated midnight scanline & list of missing scanlines
+ Sliced data
 
  """
  start_line = self.start_line if self.start_line is not None else 0
@@ -250,20 +233,17 @@ def _slice(self, data):
  along_track=data.shape[0]
  )
 
- # Slice data, update missing lines and midnight scanline to new
- # scanline range
- sliced, miss_lines, midnight_scanline = pygac.utils.slice_channel(
- data,
- start_line=start_line,
- end_line=end_line,
- first_valid_lat=first_valid_lat,
- last_valid_lat=last_valid_lat,
- midnight_scanline=self.midnight_scanline,
- miss_lines=self.missing_scanlines,
- qual_flags=self._get_qual_flags()
- )
-
- return sliced, midnight_scanline, miss_lines
+ # Slice data
+ sliced = pygac.utils.slice_channel(data,
+ start_line=start_line,
+ end_line=end_line,
+ first_valid_lat=first_valid_lat,
+ last_valid_lat=last_valid_lat)
+ if isinstance(sliced, tuple):
+ # pygac < 1.4.0
+ sliced = sliced[0]
+
+ return sliced
 
  def _get_channel(self, key):
  """Get channel and buffer results."""

satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py

@@ -117,9 +117,7 @@ def _get_fh_mocked(self, init_mock, **attrs):
  def _get_reader_mocked(self, along_track=3):
  """Create a mocked reader."""
  reader = mock.MagicMock(spacecraft_name='spacecraft_name',
- meta_data={'foo': 'bar',
- 'midnight_scanline': 1,
- 'missing_scanlines': [1, 2, 3]})
+ meta_data={'foo': 'bar'})
  reader.mask = [0, 0]
  reader.get_times.return_value = np.arange(along_track)
  reader.get_tle_lines.return_value = 'tle'
@@ -132,16 +130,26 @@ def test_init(self):
  from pygac.lac_klm import LACKLMReader
  from pygac.lac_pod import LACPODReader
 
+ kwargs = {'start_line': 1,
+ 'end_line': 2,
+ 'strip_invalid_coords': True,
+ 'interpolate_coords': True,
+ 'adjust_clock_drift': True,
+ 'tle_dir': 'tle_dir',
+ 'tle_name': 'tle_name',
+ 'tle_thresh': 123,
+ 'calibration': 'calibration'}
  for filenames, reader_cls in zip([GAC_POD_FILENAMES, GAC_KLM_FILENAMES, LAC_POD_FILENAMES, LAC_KLM_FILENAMES],
  [GACPODReader, GACKLMReader, LACPODReader, LACKLMReader]):
  for filename in filenames:
- fh = self._get_fh(filename)
+ fh = self._get_fh(filename, **kwargs)
  self.assertLess(fh.start_time, fh.end_time,
  "Start time must precede end time.")
  self.assertIs(fh.reader_class, reader_cls,
  'Wrong reader class assigned to {}'.format(filename))
 
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
+ @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.read_raw_data')
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_channel')
  def test_get_dataset_channels(self, get_channel, *mocks):
  from satpy.dataset import DatasetID
@@ -173,8 +181,6 @@ def test_get_dataset_channels(self, get_channel, *mocks):
  'orbit_number': 123,
  'sensor': 'sensor',
  'orbital_parameters': {'tle': 'tle'},
- 'midnight_scanline': 1,
- 'missing_scanlines': [1, 2, 3],
  'foo': 'bar',
  'standard_name': 'my_standard_name'})
  exp.coords['acq_time'].attrs['long_name'] = 'Mean scanline acquisition time'
@@ -187,6 +193,27 @@ def test_get_dataset_channels(self, get_channel, *mocks):
  fh.get_dataset(key=key, info={'name': 1})
  get_channel.assert_called_with(key)
 
+ def test_read_raw_data(self):
+ fh = self._get_fh_mocked(reader=None,
+ interpolate_coords='interpolate_coords',
+ creation_site='creation_site',
+ reader_kwargs={'foo': 'bar'},
+ filename='myfile')
+ reader = mock.MagicMock(mask=[0])
+ reader_cls = mock.MagicMock(return_value=reader)
+ fh.reader_class = reader_cls
+ fh.read_raw_data()
+ reader_cls.assert_called_with(interpolate_coords='interpolate_coords',
+ creation_site='creation_site',
+ foo='bar')
+ reader.read.assert_called_with('myfile')
+
+ # Test exception if all data is masked
+ reader.mask = [1]
+ fh.reader = None
+ with self.assertRaises(ValueError):
+ fh.read_raw_data()
+
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._update_attrs')
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.slice')
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_channel')
@@ -416,64 +443,59 @@ def test_strip_invalid_lat(self):
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._slice')
  def test_slice(self, _slice):
  """Test slicing."""
- def slice_patched(data):
- if len(data.shape) == 2:
- return data[1:3, :], 'midn_line', np.array([1., 2., 3.])
- return data[1:3], 'foo', np.array([0, 0, 0])
 
- _slice.side_effect = slice_patched
+ def _slice_patched(data):
+ return data[1:3]
+ _slice.side_effect = _slice_patched
+
  data = np.zeros((4, 2))
  times = np.array([1, 2, 3, 4], dtype='datetime64[us]')
 
- fh = self._get_fh_mocked()
+ fh = self._get_fh_mocked(start_line=1, end_line=3, strip_invalid_coords=False)
  data_slc, times_slc = fh.slice(data, times)
  np.testing.assert_array_equal(data_slc, data[1:3])
  np.testing.assert_array_equal(times_slc, times[1:3])
  self.assertEqual(fh.start_time, datetime(1970, 1, 1, 0, 0, 0, 2))
  self.assertEqual(fh.end_time, datetime(1970, 1, 1, 0, 0, 0, 3))
- self.assertEqual(fh.midnight_scanline, 'midn_line')
- np.testing.assert_array_equal(fh.missing_scanlines, np.array([1, 2, 3]))
- self.assertEqual(fh.missing_scanlines.dtype, int)
 
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_qual_flags')
  @mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._strip_invalid_lat')
  def test__slice(self, strip_invalid_lat, get_qual_flags):
  """Test slicing."""
  import pygac.utils
  pygac.utils.check_user_scanlines.return_value = 1, 2
- pygac.utils.slice_channel.return_value = 'sliced', 'miss_lines', 'midn_line'
+ pygac.utils.slice_channel.return_value = 'sliced'
  strip_invalid_lat.return_value = 3, 4
  get_qual_flags.return_value = 'qual_flags'
 
  data = np.zeros((2, 2))
 
  # a) Only start/end line given
- fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=False,
- midnight_scanline=None, missing_scanlines=None)
- data_slc, midn_line, miss_lines = fh._slice(data)
+ fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=False)
+ data_slc = fh._slice(data)
  self.assertEqual(data_slc, 'sliced')
- self.assertEqual(midn_line, 'midn_line')
- self.assertEqual(miss_lines, 'miss_lines')
  pygac.utils.check_user_scanlines.assert_called_with(
  start_line=5, end_line=6,
  first_valid_lat=None, last_valid_lat=None, along_track=2)
  pygac.utils.slice_channel.assert_called_with(
  data, start_line=1, end_line=2,
- first_valid_lat=None, last_valid_lat=None,
- midnight_scanline=None, miss_lines=None, qual_flags='qual_flags')
+ first_valid_lat=None, last_valid_lat=None)
 
  # b) Only strip_invalid_coords=True
- fh = self._get_fh_mocked(start_line=None, end_line=None, strip_invalid_coords=True,
- midnight_scanline=None, missing_scanlines=None)
+ fh = self._get_fh_mocked(start_line=None, end_line=None, strip_invalid_coords=True)
  fh._slice(data)
  pygac.utils.check_user_scanlines.assert_called_with(
  start_line=0, end_line=0,
  first_valid_lat=3, last_valid_lat=4, along_track=2)
 
  # c) Both
- fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=True,
- midnight_scanline=None, missing_scanlines=None)
+ fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=True)
  fh._slice(data)
  pygac.utils.check_user_scanlines.assert_called_with(
  start_line=5, end_line=6,
  first_valid_lat=3, last_valid_lat=4, along_track=2)
+
+ # Test slicing with older pygac versions
+ pygac.utils.slice_channel.return_value = ('sliced', 'foo', 'bar')
+ data_slc = fh._slice(data)
+ self.assertEqual(data_slc, 'sliced')