Support changing the S3 Endpoint

.pre-commit-config.yaml

@@ -16,6 +16,16 @@ repos:
  - id: requirements-txt-fixer
  - id: check-added-large-files
  - id: check-merge-conflict
+ - repo: https:/pycqa/isort
+ rev: 5.10.1
+ hooks:
+ - id: isort
+ name: isort (python)
+ args: [ "--profile", "black", "--filter-files" ]
+ - repo: https:/psf/black
+ rev: 22.10.0
+ hooks:
+ - id: black
  - repo: https:/pre-commit/mirrors-pylint
  rev: 'v3.0.0a5' # Use the sha / tag you want to point at
  hooks:
@@ -24,7 +34,3 @@ repos:
  rev: '5.0.4'
  hooks:
  - id: flake8
- - repo: https:/psf/black
- rev: 22.10.0
- hooks:
- - id: black

apps/cloud/odc/apps/cloud/azure_to_tar.py

@@ -1,10 +1,10 @@
 import tarfile
-import click
-from odc.azure import find_blobs, download_yamls
-
-from odc.io.tar import tar_mode, add_txt_file
 from urllib.parse import urlparse
 
+import click
+from odc.azure import download_yamls, find_blobs
+from odc.io.tar import add_txt_file, tar_mode
+
 
 @click.command("azure-to-tar")
 @click.argument("account_url", type=str, nargs=1)

apps/cloud/odc/apps/cloud/gs_to_tar.py

@@ -1,7 +1,8 @@
 import os
+import shutil
 import tarfile
+
 import click
-import shutil
 from google.cloud import storage
 
 

apps/cloud/odc/apps/cloud/redrive_to_queue.py

@@ -1,6 +1,7 @@
-import click
-import sys
 import logging
+import sys
+
+import click
 from odc.aws.queue import redrive_queue
 
 

apps/cloud/odc/apps/cloud/s3_find.py

@@ -1,6 +1,7 @@
-import click
 import sys
-from odc.aio import s3_find_glob, S3Fetcher
+
+import click
+from odc.aio import S3Fetcher, s3_find_glob
 
 
 @click.command("s3-find")
@@ -56,7 +57,7 @@ def cli(uri, skip_check, no_sign_request=None, request_payer=False):
  except ValueError as ve:
  click.echo(str(ve), err=True)
  sys.exit(1)
- except Exception as e:
+ except Exception as e: # pylint:disable=broad-except
  click.echo(str(e), err=True)
  sys.exit(1)
 

apps/cloud/odc/apps/cloud/s3_inventory.py

@@ -1,6 +1,7 @@
 import re
-from fnmatch import fnmatch
 import sys
+from fnmatch import fnmatch
+
 import click
 from odc.aws import s3_client
 from odc.aws.inventory import list_inventory

apps/cloud/odc/apps/cloud/s3_to_tar.py

@@ -1,13 +1,13 @@
-import click
-import tarfile
 import logging
 import signal
 import sys
+import tarfile
 from sys import stderr, stdout
 
+import click
 from odc.aio import S3Fetcher
 from odc.io import read_stdin_lines
-from odc.io.tar import tar_mode, add_txt_file
+from odc.io.tar import add_txt_file, tar_mode
 from odc.io.timer import RateEstimator
 
 

apps/cloud/odc/apps/cloud/thredds_to_tar.py

@@ -1,7 +1,8 @@
 import tarfile
+
 import click
-from odc.io.tar import tar_mode, add_txt_file
-from odc.thredds import thredds_find_glob, download_yamls
+from odc.io.tar import add_txt_file, tar_mode
+from odc.thredds import download_yamls, thredds_find_glob
 
 
 @click.command("thredds-to-tar")

apps/dc_tools/odc/apps/dc_tools/_stac.py

@@ -7,7 +7,6 @@
 from uuid import UUID
 
 import numpy
-
 from datacube.utils.geometry import Geometry
 from toolz import get_in
 from urlpath import URL

apps/dc_tools/odc/apps/dc_tools/add_update_products.py

@@ -6,19 +6,18 @@
 import sys
 from collections import Counter, namedtuple
 from csv import DictReader
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 import click
 import datacube
 import fsspec
 import yaml
 from datacube import Datacube
 from odc.apps.dc_tools.utils import (
- update_if_exists,
  statsd_gauge_reporting,
  statsd_setting,
+ update_if_exists,
 )
-from typing import Any, Dict, List
 
 Product = namedtuple("Product", ["name", "doc"])
 

apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py

@@ -16,13 +16,13 @@
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
  SkippedException,
+ archive_less_mature,
  bbox,
  index_update_dataset,
  limit,
- update_if_exists,
- archive_less_mature,
  statsd_gauge_reporting,
  statsd_setting,
+ update_if_exists,
 )
 from rio_stac import create_stac_item
 

apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py

@@ -15,13 +15,13 @@
 from datacube.index.hl import Doc2Dataset
 from datacube.utils import read_documents
 from odc.apps.dc_tools.utils import (
+ archive_less_mature,
  bbox,
  index_update_dataset,
  limit,
- update_if_exists,
- archive_less_mature,
  statsd_gauge_reporting,
  statsd_setting,
+ update_if_exists,
 )
 from rio_stac import create_stac_item
 

apps/dc_tools/odc/apps/dc_tools/export_md.py

@@ -9,14 +9,14 @@
  default: ['3', '4', '5']
  swir: ['1', '2']
 """
-from pathlib import Path
-import yaml
-import click
 import logging
+from pathlib import Path
 
-from datacube.testutils.io import native_geobox
-from datacube.storage import BandInfo
+import click
+import yaml
 from datacube import Datacube
+from datacube.storage import BandInfo
+from datacube.testutils.io import native_geobox
 
 _LOG = logging.getLogger(__name__)
 

apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py

@@ -1,24 +1,22 @@
 import json
+import logging
 from pathlib import Path
+from typing import Generator, Optional
 
 import click
 import datacube
+import yaml
 from datacube.index.hl import Doc2Dataset
+from odc.apps.dc_tools._stac import stac_transform
 from odc.apps.dc_tools.utils import (
- index_update_dataset,
- update_if_exists,
- archive_less_mature,
  allow_unsafe,
- transform_stac,
+ archive_less_mature,
+ index_update_dataset,
  statsd_gauge_reporting,
  statsd_setting,
+ transform_stac,
+ update_if_exists,
 )
-from odc.apps.dc_tools._stac import stac_transform
-from typing import Generator, Optional
-import logging
-
-
-import yaml
 
 logging.basicConfig(
  level=logging.WARNING,

apps/dc_tools/odc/apps/dc_tools/index_from_tar.py

@@ -6,11 +6,12 @@
 import click
 import datacube
 from datacube.utils.changes import allow_any
-from ._docs import from_yaml_doc_stream
-from ._stac import stac_transform
 from odc.io.tar import tar_doc_stream, tar_mode
 from odc.io.timer import RateEstimator
 
+from ._docs import from_yaml_doc_stream
+from ._stac import stac_transform
+
 
 def from_tar_file(tarfname, index, mk_uri, mode, doc_transform=None, **kwargs):
  """returns a sequence of tuples where each tuple is either

apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py

@@ -9,14 +9,14 @@
 import click
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
-
-
 from odc.aio import S3Fetcher, s3_find_glob
+from odc.apps.dc_tools._docs import parse_doc_stream
+from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute
 from odc.apps.dc_tools.utils import (
  IndexingException,
  SkippedException,
- archive_less_mature,
  allow_unsafe,
+ archive_less_mature,
  fail_on_missing_lineage,
  index_update_dataset,
  no_sign_request,
@@ -31,8 +31,6 @@
  update_if_exists,
  verify_lineage,
 )
-from odc.apps.dc_tools._docs import parse_doc_stream
-from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute
 
 
 # Grab the URL from the resulting S3 item

apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py

@@ -9,11 +9,11 @@
 from typing import Tuple
 
 import boto3
-from botocore import UNSIGNED
-from botocore.config import Config
 import click
 import pandas as pd
 import requests
+from botocore import UNSIGNED
+from botocore.config import Config
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
 from datacube.utils import documents
@@ -22,25 +22,26 @@
  SkippedException,
  allow_unsafe,
  archive,
+ archive_less_mature,
  fail_on_missing_lineage,
  index_update_dataset,
  limit,
  no_sign_request,
  skip_lineage,
- statsd_setting,
  statsd_gauge_reporting,
+ statsd_setting,
  transform_stac,
  transform_stac_absolute,
- archive_less_mature,
  update,
  update_if_exists,
  verify_lineage,
 )
 from odc.aws.queue import get_messages
-from ._stac import stac_transform, stac_transform_absolute
 from toolz import dicttoolz
 from yaml import safe_load
 
+from ._stac import stac_transform, stac_transform_absolute
+
 # Added log handler
 logging.basicConfig(level=logging.WARNING, handlers=[logging.StreamHandler()])
 

apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py

@@ -1,17 +1,16 @@
 """Crawl Thredds for prefixes and fetch YAML's for indexing
 and dump them into a Datacube instance
 """
-import sys
 import logging
-from typing import Tuple
+import sys
+from typing import List, Tuple
 
 import click
-from odc.thredds import thredds_find_glob, download_yamls
-from odc.apps.dc_tools.utils import statsd_gauge_reporting, statsd_setting
-from ._docs import from_yaml_doc_stream
 from datacube import Datacube
+from odc.apps.dc_tools.utils import statsd_gauge_reporting, statsd_setting
+from odc.thredds import download_yamls, thredds_find_glob
 
-from typing import List, Tuple
+from ._docs import from_yaml_doc_stream
 
 
 def dump_list_to_odc(

apps/dc_tools/odc/apps/dc_tools/utils.py

@@ -1,16 +1,13 @@
-import os
 import logging
-import click
-import pkg_resources
-
+import os
 from typing import Iterable, Optional, Union
 
+import click
+import pkg_resources
 from datacube import Datacube
 from datacube.index.hl import Doc2Dataset
 from datacube.utils import changes
-
-from datadog import statsd, initialize
-
+from datadog import initialize, statsd
 
 ESRI_LANDCOVER_BASE_URI = (
  "https://ai4edataeuwest.blob.core.windows.net/io-lulc/"

apps/dc_tools/tests/conftest.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+
 import pytest
 import yaml
 from datacube import Datacube

apps/dc_tools/tests/test_add_update_products.py

@@ -1,11 +1,10 @@
 from pathlib import Path
 
 import pytest
-from datacube import Datacube
 from click.testing import CliRunner
+from datacube import Datacube
 from odc.apps.dc_tools.add_update_products import _get_product, _parse_csv, cli
 
-
 TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data")
 LOCAL_EXAMPLE: str = "example_product_list.csv"
 PRODUCT_EXAMPLE: str = (

apps/dc_tools/tests/test_cop_dem_to_dc.py

@@ -1,9 +1,6 @@
 import pytest
-
 from click.testing import CliRunner
-
-from odc.apps.dc_tools.cop_dem_to_dc import get_dem_tile_uris, cli
-
+from odc.apps.dc_tools.cop_dem_to_dc import cli, get_dem_tile_uris
 
 PRODUCTS = ["cop_30", "cop_90"]