Merge pull request #2 from pytroll/main

Merge main into read_error

.github/workflows/ci.yaml

@@ -38,11 +38,11 @@ jobs:
  - name: Setup Conda Environment
  uses: conda-incubator/setup-miniconda@v3
  with:
- miniforge-variant: Mambaforge
  miniforge-version: latest
- use-mamba: true
  python-version: ${{ matrix.python-version }}
  activate-environment: test-environment
+ mamba-version: "*"
+ channels: conda-forge
 
  - name: Set cache environment variables
  shell: bash -l {0}
@@ -80,7 +80,7 @@ jobs:
  numpy \
  pandas \
  scipy
- conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa
+ conda remove --force-remove -y pykdtree pyresample python-geotiepoints pyhdf netcdf4 h5py cftime astropy pyerfa || true
  python -m pip install --upgrade --no-deps --pre --no-build-isolation \
  pyerfa \
  git+https:/storpipfugl/pykdtree \

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ fail_fast: false
 repos:
  - repo: https:/astral-sh/ruff-pre-commit
  # Ruff version.
- rev: 'v0.5.6'
+ rev: 'v0.6.3'
  hooks:
  - id: ruff
  - repo: https:/pre-commit/pre-commit-hooks
@@ -19,7 +19,7 @@ repos:
  - id: bandit
  args: [--ini, .bandit]
  - repo: https:/pre-commit/mirrors-mypy
- rev: 'v1.11.1' # Use the sha / tag you want to point at
+ rev: 'v1.11.2' # Use the sha / tag you want to point at
  hooks:
  - id: mypy
  additional_dependencies:

.readthedocs.yml

@@ -18,5 +18,7 @@ build:
  - git fetch --tags
  pre_install:
  - git update-index --assume-unchanged doc/rtd_environment.yml doc/source/conf.py
+ pre_build:
+ - cd doc/source && if [ "$READTHEDOCS_VERSION" = "latest" ] || [ "$READTHEDOCS_VERSION" = "stable" ]; then python generate_area_def_list.py; else touch area_def_list.rst; fi
 conda:
  environment: doc/rtd_environment.yml

AUTHORS.md

@@ -77,6 +77,7 @@ The following people have made contributions to this project:
 - Marco Sassi - meteoswiss
 - [Stefan Scheiblauer (StefanSnippetCoder)](https:/StefanSnippetCoder)
 - [Ronald Scheirer](https:/)
+- [Michael Schmutz (Graenni)](https:/Graenni) - Meteotest AG
 - [Hauke Schulz (observingClouds)](https:/observingClouds)
 - [Jakub Seidl (seidlj)](https:/seidlj)
 - [Eysteinn Sigurðsson (eysteinn)](https:/eysteinn)

continuous_integration/environment.yaml

@@ -64,3 +64,4 @@ dependencies:
  - trollimage>=1.24
  - pyspectral
  - pyorbital
+ - pyPublicDecompWT

doc/rtd_environment.yml

@@ -31,7 +31,7 @@ dependencies:
  - xarray
  - zarr
  - xarray-datatree
- - cartopy
+ - geoviews
  - pip:
  - graphviz
  - pytest-lazy-fixtures

doc/source/conf.py

@@ -24,15 +24,9 @@
 sys.path.append(os.path.abspath("../../"))
 sys.path.append(os.path.abspath(os.path.dirname(__file__)))
 
-from pyresample.area_config import ( # noqa: E402
- _create_area_def_from_dict,
- _read_yaml_area_file_content,
- generate_area_def_rst_list,
-)
-from reader_table import generate_reader_table, rst_table_header, rst_table_row # noqa: E402
+from reader_table import generate_reader_table # noqa: E402
 
 import satpy # noqa: E402
-from satpy.resample import get_area_file # noqa: E402
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -86,26 +80,6 @@ def __getattr__(cls, name):
 with open("reader_table.rst", mode="w") as f:
  f.write(generate_reader_table())
 
-# create table from area definition yaml file
-area_file = get_area_file()[0]
-
-area_dict = _read_yaml_area_file_content(area_file)
-area_table = [rst_table_header("Area Definitions", header=["Name", "Description", "Projection"],
- widths="auto", class_name="area-table")]
-
-for aname, params in area_dict.items():
- area = _create_area_def_from_dict(aname, params)
- if not hasattr(area, "_repr_html_"):
- continue
-
- area_table.append(rst_table_row([f"`{aname}`_", area.description,
- area.proj_dict.get("proj")]))
-
-with open("area_def_list.rst", mode="w") as f:
- f.write("".join(area_table))
- f.write("\n\n")
- f.write(generate_area_def_rst_list(area_file))
-
 # -- General configuration -----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions

doc/source/dev_guide/index.rst

@@ -128,6 +128,15 @@ additional packages are needed. These can be installed with ::
 
  pip install -e ".[doc]"
 
+Generating the documentation requires a one-time script to generate a list
+of previews of all of the AreaDefinition objects used by the documentation.
+This script can take 2+ minutes to execute so it is run separately from the
+normal documentation build process. To run it::
+
+ cd doc/source/
+ python generate_area_def_list.py
+ cd ../../
+
 After editing the source files there the documentation can be generated locally::
 
  cd doc

doc/source/docutils.conf

@@ -0,0 +1,2 @@
+[parsers]
+line_length_limit=30000000

doc/source/generate_area_def_list.py

@@ -0,0 +1,152 @@
+"""Generate the area definition list restructuredtext document.
+
+This should be run once before generating the sphinx documentation to
+produce the ``area_def_list.rst`` file referenced by ``satpy/resample.py``.
+
+"""
+import logging
+import pathlib
+import sys
+from datetime import datetime
+
+import bokeh
+import geoviews as gv
+import geoviews.feature as gf
+from bokeh.embed import components
+from jinja2 import Template
+from pyresample._formatting_html import _load_static_files
+from pyresample.area_config import area_repr, load_area
+from pyresample.utils.proj4 import ignore_pyproj_proj_warnings
+from reader_table import rst_table_header, rst_table_row
+
+from satpy.resample import get_area_file
+
+logger = logging.getLogger(__name__)
+
+gv.extension("bokeh")
+
+
+TEMPLATE = '''
+
+{{ table_header }}
+{% for area_name, area_def in areas.items() if area_def._repr_html_ is defined %}
+{{ create_table_row(area_name, area_def) }}
+{% endfor %}
+
+
+.. raw:: html
+
+ {{ resources }}
+ {{ pyr_icons_svg | indent(5) }}
+ <style>
+ {{ pyr_css_style | indent(5) }}
+ </style>
+ {{ script | indent(5)}}
+
+{% for area_name, area_div in area_divs_dict.items() %}
+
+{{ area_name }}
+{{ rst_underline('^', area_name|length) }}
+
+.. raw:: html
+
+ {{ area_repr(areas[area_name], map_content=area_div, include_header=False, include_static_files=False) |
+ indent(5) }}
+ <br>
+
+{% endfor %}
+''' # noqa: Q001
+
+
+def main():
+ """Parse CLI arguments and generate area definition list file."""
+ from argparse import ArgumentParser
+
+ parser = ArgumentParser(description="Generate restructuredtext area definition list for sphinx documentation")
+ parser.add_argument("--area-file",
+ help="Input area YAML file to read")
+ parser.add_argument("-o", "--output-file",
+ type=pathlib.Path,
+ help="HTML or restructuretext filename to create. "
+ "Defaults to 'area_def_list.rst' in the "
+ "documentation source directory.")
+ args = parser.parse_args()
+ logging.basicConfig(level=logging.INFO)
+
+ if args.output_file is None:
+ args.output_file = str(pathlib.Path(__file__).resolve().parent / "area_def_list.rst")
+ area_file = args.area_file
+ if area_file is None:
+ area_file = get_area_file()[0]
+
+ area_list = load_area(area_file)
+ areas_dict = {_area_name(area): area for area in area_list}
+ logger.info(f"Generating bokeh plots ({datetime.now()})...")
+ script, divs_dict = _generate_html_map_divs(areas_dict)
+ logger.info(f"Done generating bokeh plots ({datetime.now()})")
+
+ def rst_underline(ch, num_chars):
+ return ch * num_chars
+
+ template = Template(TEMPLATE)
+ icons_svg, css_style = _load_static_files()
+ logger.info(f"Rendering document ({datetime.now()})...")
+ res = template.render(
+ resources=bokeh.resources.CDN.render(),
+ script=script,
+ area_divs_dict=divs_dict,
+ areas=areas_dict,
+ rst_underline=rst_underline,
+ area_repr=area_repr,
+ pyr_icons_svg=icons_svg,
+ pyr_css_style=css_style,
+ table_header=rst_table_header("Area Definitions", header=["Name", "Description", "Projection"],
+ widths="auto", class_name="area-table"),
+ create_table_row=_area_table_row,
+ )
+ logger.info(f"Done rendering document ({datetime.now()})")
+
+ with open(args.output_file, mode="w") as f:
+ f.write(res)
+
+
+def _area_name(area_def) -> str:
+ if hasattr(area_def, "attrs"):
+ # pyresample 2
+ return area_def.attrs["name"]
+ # pyresample 1
+ return area_def.area_id
+
+
+def _area_table_row(area_name, area_def):
+ with ignore_pyproj_proj_warnings():
+ area_proj = area_def.proj_dict.get("proj")
+ return rst_table_row([f"`{area_name}`_", area_def.description, area_proj])
+
+
+def _generate_html_map_divs(areas_dict: dict) -> tuple[str, dict]:
+ areas_bokeh_models = {}
+ for area_name, area_def in areas_dict.items():
+ if not hasattr(area_def, "to_cartopy_crs"):
+ logger.info(f"Skipping {area_name} because it can't be converted to cartopy CRS")
+ continue
+ crs = area_def.to_cartopy_crs()
+
+ features = gv.Overlay([gf.ocean, gf.land, gf.borders, gf.coastline])
+ f = gv.render(
+ features.opts(
+ toolbar=None,
+ default_tools=[],
+ projection=crs,
+ xlim=crs.bounds[:2],
+ ylim=crs.bounds[2:],
+ ),
+ backend="bokeh")
+ areas_bokeh_models[area_name] = f
+
+ script, divs_dict = components(areas_bokeh_models)
+ return script, divs_dict
+
+
+if __name__ == "__main__":
+ sys.exit(main())

doc/source/reading.rst

@@ -305,6 +305,13 @@ time etc. The following attributes are standardized across all readers:
  should happen and when they actually do. See :ref:`time_metadata` below for
  details.
 * ``raw_metadata``: Raw, unprocessed metadata from the reader.
+* ``rows_per_scan``: Optional integer indicating how many rows of data
+ represent a single scan of the instrument. This is primarily used by
+ some resampling algorithms (ex. EWA) to produce better results and only
+ makes sense for swath-based (usually polar-orbiting) instruments. For
+ example, MODIS 1km data has 10 rows of data per scan. If an instrument
+ does not have multiple rows per scan this should usually be set to 0 rather
+ than 1 to indicate that the entire swath should be treated as a whole.
 
 Note that the above attributes are not necessarily available for each dataset.
 

pyproject.toml

@@ -55,7 +55,7 @@ msi_safe = ["rioxarray", "bottleneck", "python-geotiepoints", "defusedxml"]
 nc_nwcsaf_msg = ["netCDF4 >= 1.1.8"]
 sar_c = ["python-geotiepoints >= 1.1.7", "rasterio", "rioxarray", "defusedxml"]
 abi_l1b = ["h5netcdf"]
-seviri_l1b_hrit = ["pyorbital >= 1.3.1"]
+seviri_l1b_hrit = ["pyorbital >= 1.3.1", "pyPublicDecompWT"]
 seviri_l1b_native = ["pyorbital >= 1.3.1"]
 seviri_l1b_nc = ["pyorbital >= 1.3.1", "netCDF4 >= 1.1.8"]
 seviri_l2_bufr = ["eccodes"]

satpy/etc/enhancements/generic.yaml

@@ -1192,6 +1192,18 @@ enhancements:
  factor: 21.0
  min_stretch: 0.0
  max_stretch: 20.0
+ chlor_a_bgc:
+ name: chlor_a
+ reader: oci_l2_bgc
+ operations:
+ - name: stretch
+ method: !!python/name:satpy.enhancements.stretch
+ kwargs:
+ stretch: log
+ base: "10"
+ factor: 21.0
+ min_stretch: 0.0
+ max_stretch: 20.0
 
  cimss_cloud_type:
  standard_name: cimss_cloud_type

satpy/etc/readers/mcd12q1.yaml

@@ -0,0 +1,49 @@
+reader:
+ name: mcd12q1
+ short_name: MCD12Q1
+ long_name: MODIS Level 3 (mcd12Q1) data in HDF-EOS format
+ description: MODIS HDF-EOS MCD12Q1 L3 Reader
+ status: Beta
+ supports_fsspec: false
+ reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
+ sensors: [modis]
+
+file_types:
+ modis_mcd12q1_hdf_eos:
+ file_patterns: ['MCD12Q1.A{start_time:%Y%j}.{tile_id}.{collection:03d}.{production_time:%Y%j%H%M%S}.hdf']
+ file_reader: !!python/name:satpy.readers.mcd12q1.MCD12Q1HDFFileHandler
+
+datasets:
+ LC_Type1:
+ name: LC_Type1
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+
+ LC_Type2:
+ name: LC_Type2
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Type3:
+ name: LC_Type3
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Type4:
+ name: LC_Type4
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Type5:
+ name: LC_Type5
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Prop1:
+ name: LC_Prop1
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Prop2:
+ name: LC_Prop2
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos
+ LC_Prop3:
+ name: LC_Prop3
+ resolution: 500
+ file_type: modis_mcd12q1_hdf_eos