Promote soil-texture collections to production

[anayeaye]

What

This issue is to collect the remaining data tasks required to publish the soil-texture datasets to the earthdata dashboard.

Blocked PR NASA-IMPACT/veda-config#446
Related veda-config dataset in develop https:/NASA-IMPACT/veda-config/blob/develop/datasets/soil-texture.data.mdx?plain=1

Tasks

• Finish validating and updating soil-texture collections in staging backend
• Transfer files to production
• Publish collections to production
• Open veda-data PR with validated publication configuration

Raw config info

there should be 7 collections that follow this pattern

{
  "collection": "soil-texture-0cm",
  "title": "ISRIC World Soil Texture Classification (0 cm)",
  "description": "ISRIC World Soil Texture Classification at the surface level (0 cm). Broken into 12 classes of 1:Cl, 2:SiCl, 3:SaCl, 4:ClLo, 5:SiClLo, 6:SaClLo, 7:Lo, 8:SiLo, 9:SaLo, 10:Si, 11:LoSa, 12:Sa.",
  "license": "CC0-1.0",
  "is_periodic": true,
  "time_density": "year",
  "spatial_extent": {
    "xmin": -180,
    "ymin": -90,
    "xmax": 180,
    "ymax": 90
  },
  "temporal_extent": {
    "startdate": "2017-01-01T00:00:00Z",
    "enddate": "2017-12-31T23:59:59Z"
  },
  "sample_files": [
    "s3://veda-data-store-staging/soil/soil_texture_0cm_250m_cog_2017.tif" 
  ],
  "discovery_items": [
    {
      "discovery": "s3",
      "cogify": false,
      "upload": false,
      "dry_run": false,
      "prefix": "soil/",
      "bucket": "veda-data-store-staging",
      "filename_regex": "(.*)soil_texture_0cm_250m_cog_2017.tif$",
      "use_multithreading": false
    }
  ],
  "data_type": "cog",
  "datetime_range": "year"
}

[anayeaye]

Thumbnail from veda-config uploaded to https://thumbnails.openveda.cloud/soil-texture-background.jpeg

"assets": {
    "thumbnail": {
        "title": "Thumbnail",
        "description": "Photo by [Soil Sensor](https://soilsensor.com/articles/soil-textures/) (Examples of two different soil types)",
        "href": "https://thumbnails.openveda.cloud/soil-texture-background.jpeg",
        "type": "image/jpeg",
        "roles": ["thumbnail"]
    }
}

[anayeaye]

Also I have filled out more of the dataset publish configuration and consolidated into a single multi-asset collection.

{
    "collection": "soil-texture-0cm",
    "title": "ISRIC World Soil Texture Classification (0 cm)",
    "description": "ISRIC World Soil Texture Classification at the surface level (0 cm). Broken into 12 classes of 1:Cl, 2:SiCl, 3:SaCl, 4:ClLo, 5:SiClLo, 6:SaClLo, 7:Lo, 8:SiLo, 9:SaLo, 10:Si, 11:LoSa, 12:Sa.",
    "license": "CC0-1.0",
    "providers": [
        {
        "name": "NASA VEDA",
        "roles": [
            "host"
        ],
        "url": "https://www.earthdata.nasa.gov/dashboard/"
        }
    ],
    "stac_version": "1.0.0",
    "stac_extensions": [
        "https://stac-extensions.github.io/render/v1.0.0/schema.json",
        "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
    ],
    "is_periodic": true,
    "time_density": "year",
    "spatial_extent": {
      "xmin": -180,
      "ymin": -90,
      "xmax": 180,
      "ymax": 90
    },
    "temporal_extent": {
      "startdate": "2017-01-01T00:00:00Z",
      "enddate": "2017-12-31T23:59:59Z"
    },
    "sample_files": [
      "s3://veda-data-store-staging/soil/soil_texture_0cm_250m_cog_2017.tif" 
    ],
    "assets": {
        "thumbnail": {
            "title": "Thumbnail",
            "description": "Photo by [Soil Sensor](https://soilsensor.com/articles/soil-textures/) (Examples of two different soil types)",
            "href": "https://thumbnails.openveda.cloud/soil-texture-background.jpeg",
            "type": "image/jpeg",
            "roles": ["thumbnail"]
        }
    },
    "item_assets": {
        "soil_texture_0cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (0 cm)",
            "description": "ISRIC World Soil Texture Classification (0 cm)"
        },
        "soil_texture_5cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (5 cm)",
            "description": "ISRIC World Soil Texture Classification (5 cm)"
        },
        "soil_texture_15cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (15cm)",
            "description": "ISRIC World Soil Texture Classification (15 cm)"
        },
        "soil_texture_30cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (30 cm)",
            "description": "ISRIC World Soil Texture Classification (30 cm)"
        },
        "soil_texture_60cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (60 cm)",
            "description": "ISRIC World Soil Texture Classification (60 cm)"
        },
        "soil_texture_100cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (100 cm)",
            "description": "ISRIC World Soil Texture Classification (100 cm)"
        },
        "soil_texture_200cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (200 cm)",
            "description": "ISRIC World Soil Texture Classification (200 cm)"
        }
    },    
    "discovery_items": [
      {
        "discovery": "s3",
        "dry_run": false,
        "prefix": "soil/",
        "bucket": "veda-data-store-staging",
        "filename_regex": "(.*)soil_texture_(.*)_250m_cog_2017.tif$",
        "assets": {
            "soil_texture_0cm_250m": {
                "title": "Soil Texture Class (0 cm)",
                "description": "ISRIC World Soil Texture Classification (0 cm)",
                "regex": "(.*)soil_texture_0cm_250m_cog_2017.tif$"
            },
            "soil_texture_5cm_250m": {
                "title": "Soil Texture Class (5 cm)",
                "description": "ISRIC World Soil Texture Classification (5 cm)",
                "regex": "(.*)soil_texture_5cm_250m_cog_2017.tif$"
            },
            "soil_texture_15cm_250m": {
                "title": "Soil Texture Class (15 cm)",
                "description": "ISRIC World Soil Texture Classification (15 cm)",
                "regex": "(.*)soil_texture_15cm_250m_cog_2017.tif$"
            },
            "soil_texture_30cm_250m": {
                "title": "Soil Texture Class (30 cm)",
                "description": "ISRIC World Soil Texture Classification (30 cm)",
                "regex": "(.*)soil_texture_30cm_250m_cog_2017.tif$"
            },
            "soil_texture_60cm_250m": {
                "title": "Soil Texture Class (60 cm)",
                "description": "ISRIC World Soil Texture Classification (60 cm)",
                "regex": "(.*)soil_texture_60cm_250m_cog_2017.tif$"
            },
            "soil_texture_100cm_250m": {
                "title": "Soil Texture Class (100 cm)",
                "description": "ISRIC World Soil Texture Classification (100 cm)",
                "regex": "(.*)soil_texture_100cm_250m_cog_2017.tif$"
            },
            "soil_texture_200cm_250m": {
                "title": "Soil Texture Class (200 cm)",
                "description": "ISRIC World Soil Texture Classification (200 cm)",
                "regex": "(.*)soil_texture_0cm_250m_cog_2017.tif$"
            }
        }
      }
    ],
    "data_type": "cog",
    "datetime_range": "year"
  }

Still needed for the staging publish testing

• provider info for ISRIC https://www.isric.org
• renders for these COGs (https://openveda.cloud/api/raster/cog/viewer?url=s3://veda-data-store-staging/soil/soil_texture_0cm_250m_cog_2017.tif)

[anayeaye]

We got a test run of this through the staging ingest API by adding optional parameters including id_regex this time (maybe multi-asset config only but this didn't seem to be required last week). There are some test items so we will need to come back and clean up the configuration a bit more.

It also looks like this custom colormap is going to be too long for the production firewall. This works in staging but not in production. Backend issue NASA-IMPACT/veda-backend#441

https://staging.openveda.cloud/api/raster/cog/preview.png?url=s3%3A%2F%2Fveda-data-store-staging%2Fsoil%2Fsoil_texture_60cm_250m_cog_2017.tif&nodata=255&colormap=%7B%221%22%3A+%22%23F89E61%22%2C+%222%22%3A+%22%23BA8560%22%2C+%223%22%3A+%22%23D8D2B4%22%2C+%224%22%3A+%22%23AE734C%22%2C+%225%22%3A+%22%239E8478%22%2C+%226%22%3A+%22%23C6A365%22%2C+%227%22%3A+%22%23B4A67D%22%2C+%228%22%3A+%22%23E1D4C4%22%2C+%229%22%3A+%22%23BEB56D%22%2C+%2210%22%3A+%22%23777C7A%22%2C+%2211%22%3A+%22%23A89B6F%22%2C+%2212%22%3A+%22%23E9E2AF%22%7D&resampling=nearest&return_mask=True

**NOTE: multi asset config still needs work!!!
I think there is further tuning to do for the id_regex parameter in discovery config. We should have one item with 7 assets (one for layer). Right now we are getting individual items for 6 or the 7 COGs in s3.

{
    "collection": "soil-texture",
    "title": "ISRIC World Soil Texture Classification",
    "description": "ISRIC World Soil Texture Classification at the surface level. Broken into 12 classes of 1:Cl, 2:SiCl, 3:SaCl, 4:ClLo, 5:SiClLo, 6:SaClLo, 7:Lo, 8:SiLo, 9:SaLo, 10:Si, 11:LoSa, 12:Sa.",
    "license": "CC0-1.0",
    "providers": [
        {
        "name": "NASA VEDA",
        "roles": [
            "host"
        ],
        "url": "https://www.earthdata.nasa.gov/dashboard/"
        }
    ],
    "stac_version": "1.0.0",
    "stac_extensions": [
        "https://stac-extensions.github.io/render/v1.0.0/schema.json",
        "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
    ],
    "is_periodic": true,
    "time_density": "year",
    "spatial_extent": {
      "xmin": -180,
      "ymin": -90,
      "xmax": 180,
      "ymax": 90
    },
    "temporal_extent": {
      "startdate": "2017-01-01T00:00:00Z",
      "enddate": "2017-12-31T23:59:59Z"
    },
    "sample_files": [
      "s3://veda-data-store-staging/soil/soil_texture_0cm_250m_cog_2017.tif" 
    ],
    "assets": {
        "thumbnail": {
            "title": "Thumbnail",
            "description": "Photo by [Soil Sensor](https://soilsensor.com/articles/soil-textures/) (Examples of two different soil types)",
            "href": "https://thumbnails.openveda.cloud/soil-texture-background.jpeg",
            "type": "image/jpeg",
            "roles": ["thumbnail"]
        }
    },
    "item_assets": {
        "soil_texture_0cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (0 cm)",
            "description": "ISRIC World Soil Texture Classification (0 cm)"
        },
        "soil_texture_5cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (5 cm)",
            "description": "ISRIC World Soil Texture Classification (5 cm)"
        },
        "soil_texture_15cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (15cm)",
            "description": "ISRIC World Soil Texture Classification (15 cm)"
        },
        "soil_texture_30cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (30 cm)",
            "description": "ISRIC World Soil Texture Classification (30 cm)"
        },
        "soil_texture_60cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (60 cm)",
            "description": "ISRIC World Soil Texture Classification (60 cm)"
        },
        "soil_texture_100cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (100 cm)",
            "description": "ISRIC World Soil Texture Classification (100 cm)"
        },
        "soil_texture_200cm_250m": {
            "type": "image/tiff; application=geotiff; profile=cloud-optimized",
            "roles": [
                "data",
                "layer"
            ],
            "title": "Soil Texture Class (200 cm)",
            "description": "ISRIC World Soil Texture Classification (200 cm)"
        }
    },    
    "discovery_items": [
      {
        "discovery": "s3",
        "prefix": "soil/",
        "bucket": "veda-data-store-staging",
        "filename_regex": "^(.*)soil_texture_(.*)_250m_cog_2017.tif$",
        "id_template": "{}",
        "id_regex":"(.*).tif$",
        "assets": {
            "soil_texture_0cm_250m": {
                "title": "Soil Texture Class (0 cm)",
                "description": "ISRIC World Soil Texture Classification (0 cm)",
                "regex": "^(.*)soil_texture_0cm_250m_cog_2017.tif$"
            },
            "soil_texture_5cm_250m": {
                "title": "Soil Texture Class (5 cm)",
                "description": "ISRIC World Soil Texture Classification (5 cm)",
                "regex": "^(.*)soil_texture_5cm_250m_cog_2017.tif$"
            },
            "soil_texture_15cm_250m": {
                "title": "Soil Texture Class (15 cm)",
                "description": "ISRIC World Soil Texture Classification (15 cm)",
                "regex": "^(.*)soil_texture_15cm_250m_cog_2017.tif$"
            },
            "soil_texture_30cm_250m": {
                "title": "Soil Texture Class (30 cm)",
                "description": "ISRIC World Soil Texture Classification (30 cm)",
                "regex": "^(.*)soil_texture_30cm_250m_cog_2017.tif$"
            },
            "soil_texture_60cm_250m": {
                "title": "Soil Texture Class (60 cm)",
                "description": "ISRIC World Soil Texture Classification (60 cm)",
                "regex": "^(.*)soil_texture_60cm_250m_cog_2017.tif$"
            },
            "soil_texture_100cm_250m": {
                "title": "Soil Texture Class (100 cm)",
                "description": "ISRIC World Soil Texture Classification (100 cm)",
                "regex": "^(.*)soil_texture_100cm_250m_cog_2017.tif$"
            },
            "soil_texture_200cm_250m": {
                "title": "Soil Texture Class (200 cm)",
                "description": "ISRIC World Soil Texture Classification (200 cm)",
                "regex": "^(.*)soil_texture_0cm_250m_cog_2017.tif$"
            }
        }
      }
    ],
    "data_type": "cog",
    "datetime_range": "year",
    "renders": {
        "dashboard": {
            "resampling": "nearest",
            "nodata": 255,
            "colormap": {
                "1": "#F89E61", 
                "2": "#BA8560", 
                "3": "#D8D2B4", 
                "4": "#AE734C", 
                "5": "#9E8478", 
                "6": "#C6A365",
                "7": "#B4A67D", 
                "8": "#E1D4C4", 
                "9": "#BEB56D", 
                "10": "#777C7A", 
                "11": "#A89B6F", 
                "12": "#E9E2AF"
            },
            "bidx": [
                1
            ],
            "return_mask": true,
            "assets": [
                "soil_texture_0cm_250m"
            ],
            "title": "VEDA Dashboard Render Parameters"
        }
    }
  }