[2/3] Tag AWS resources created with Terraform (#1552)

.gitlab-ci.yml

@@ -60,6 +60,7 @@ deploy:
  stage: deploy
  script:
  - terraform version
+ - make -C terraform check_schema
  - make package
  - make auto_deploy
  - make create

README.md

@@ -367,9 +367,9 @@ the bucket, you may want to include the region name at then end of the bucket
 name. That way you can have consistent bucket names across regions.
 
 Create a Route 53 hosted zone for the Azul service and indexer. Multiple
-deployments  can share a hosted zone but they don't have to. The name of the
+deployments can share a hosted zone but they don't have to. The name of the
 hosted zone is configured with `AZUL_DOMAIN_NAME`. `make deploy` will
-automatically provision record sets in  the configured zone but it will not
+automatically provision record sets in the configured zone but it will not
 create the zone itself or register the domain name it is associated with.
 
 Optionally create another hosted zone for the URL shortener. The URLs produced
@@ -379,6 +379,10 @@ supported to use the same zone for both `AZUL_URL_REDIRECT_BASE_DOMAIN_NAME` and
 `AZUL_DOMAIN_NAME` but this was not tested. The shortener zone can be a
 subdomain of the main Azul zone but it doesn't have to be.
 
+The hosted zone(s) should be configured with tags for cost tracking. A list of
+tags that should be provisioned is noted in
+[src/azul/deployment.py:tags](src/azul/deployment.py).
+
 If you intend to set up a Gitlab instance for CI/CD of your Azul deployments, an
 EBS volume needs to be created as well. See [gitlab.tf.json.template.py] and the
 [section on CI/CD](#9-continuous-deployment-and-integration) and for details.

scripts/prepare_lambda_deployment.py

@@ -6,20 +6,32 @@
 from pathlib import (
  Path,
 )
+import re
 import shutil
 import sys
+import typing
 
 from azul import (
  config,
 )
+from azul.deployment import (
+ populate_tags,
+)
 from azul.files import (
  write_file_atomically,
 )
 from azul.logging import (
  configure_script_logging,
 )
+from azul.types import (
+ AnyJSON,
+ JSON,
+ PrimitiveJSON,
+)
 
 log = logging.getLogger(__name__)
+T = typing.TypeVar('T', *PrimitiveJSON.__args__) # noqa
+U = typing.TypeVar('U', *AnyJSON.__args__) # noqa
 
 
 def transform_tf(input_json):
@@ -66,6 +78,87 @@ def patch_cloudwatch_resource(resource_type_name, property_name):
  return input_json
 
 
+def patch_resource_names(tf_config: JSON) -> JSON:
+ """
+ Some Chalice-generated resources have named ending in `-event`. The
+ dash prevents generation of a fully qualified resource name (i.e.,
+ with `config.qualified_resource_name`. This function returns
+ Terraform configuration with names and references updated to
+ remove the trailing `-event`.
+
+ >>> from azul.doctests import assert_json
+ >>> assert_json(patch_resource_names({
+ ... 'resource': {
+ ... 'aws_cloudwatch_event_rule': {
+ ... 'indexercachehealth-event': {
+ ... 'foo': 'abc'
+ ... },
+ ... 'servicecachehealth-event': {
+ ... 'bar': '${aws_cloudwatch_event_rule.indexercachehealth-event.foo}'
+ ... }
+ ... }
+ ... }
+ ... }))
+ {
+ "resource": {
+ "aws_cloudwatch_event_rule": {
+ "indexercachehealth": {
+ "foo": "abc"
+ },
+ "servicecachehealth": {
+ "bar": "${aws_cloudwatch_event_rule.indexercachehealth.foo}"
+ }
+ }
+ }
+ }
+
+
+ >>> assert_json(patch_resource_names({
+ ... 'resource': {
+ ... 'unaffected_resource_type': {
+ ... 'unaffected_resource_name': {}
+ ... }
+ ... }
+ ... }))
+ {
+ "resource": {
+ "unaffected_resource_type": {
+ "unaffected_resource_name": {}
+ }
+ }
+ }
+ """
+
+ def _patch_name(resource_name: str) -> str:
+ return re.sub(r'(?!\w+\.)?(\w+)(?:-event)(?=\.\w+)?', r'\1', resource_name)
+
+ def _replace_refs(value: T) -> T:
+ if isinstance(value, str):
+ return re.sub(r'(?!\${)((\w|-|\.)+)(?=})',
+ lambda m: _patch_name(m.group(1)),
+ value)
+ else:
+ return value
+
+ def _patch_refs(block: U) -> U:
+ assert not isinstance(block, typing.Sequence)
+ return {
+ k: _patch_refs(v) if isinstance(v, typing.Mapping) else _replace_refs(v)
+ for k, v in block.items()
+ }
+
+ return {
+ k: v if k != 'resource' else {
+ resource_type: {
+ _patch_name(resource_name): _patch_refs(arguments)
+ for resource_name, arguments in resource.items()
+ }
+ for resource_type, resource in v.items()
+ }
+ for k, v in tf_config.items()
+ }
+
+
 def main(argv):
  parser = ArgumentParser(
  description='Prepare the Terraform config generated by `chalice package'
@@ -89,7 +182,7 @@ def main(argv):
  log.info('Transforming %s to %s', tf_src, tf_dst)
  with open(tf_src, 'r') as f:
  output_json = json.load(f)
- output_json = transform_tf(output_json)
+ output_json = populate_tags(patch_resource_names(transform_tf(output_json)))
  with write_file_atomically(tf_dst) as f:
  json.dump(output_json, f, indent=4)
 

scripts/rename_resources.py

@@ -34,6 +34,16 @@
  'null_resource.hmac-secret': 'null_resource.hmac_secret'
 }
 
+rename_chalice = [
+ 'module.chalice_service.aws_cloudwatch_event_rule.indexercachehealth-event',
+ 'module.chalice_service.aws_cloudwatch_event_target.indexercachehealth-event',
+ 'module.chalice_service.aws_lambda_permission.indexercachehealth-event',
+ 'module.chalice_service.aws_cloudwatch_event_rule.servicecachehealth-event',
+ 'module.chalice_service.aws_cloudwatch_event_target.servicecachehealth-event',
+ 'module.chalice_service.aws_lambda_permission.servicecachehealth-event'
+]
+renamed.update({name: name.split('-')[0] for name in rename_chalice})
+
 
 def terraform_state(command: str, *args: str) -> bytes:
  proc = subprocess.run(['terraform', 'state', command, *args],

scripts/terraform_schema.py

@@ -0,0 +1,42 @@
+"""
+Save Terraform version and schema information to the file specified
+in AZUL_TRACKED_SCHEMA_PATH or verify that the Terraform information
+in that file is up-to-date.
+"""
+import argparse
+import logging
+
+from azul import (
+ config,
+)
+from azul.deployment import (
+ terraform,
+)
+from azul.logging import (
+ configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+
+def check_schema_json() -> None:
+ if terraform.tracked_versions == terraform.versions():
+ return
+ else:
+ raise RuntimeError('Tracked Terraform schema is out of date. Run `make -C '
+ f'terraform schema` and commit {config.tracked_terraform_schema}')
+
+
+if __name__ == '__main__':
+ configure_script_logging()
+ commands = {
+ 'generate': terraform.write_tracked_schema,
+ 'check': check_schema_json
+ }
+ # https://youtrack.jetbrains.com/issue/PY-41806
+ # noinspection PyTypeChecker
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('command', choices=commands)
+ arguments = parser.parse_args()
+ commands[arguments.command]()

src/azul/__init__.py

@@ -1,6 +1,9 @@
 import functools
 import logging
 import os
+from pathlib import (
+ Path,
+)
 import re
 from typing import (
  ClassVar,
@@ -149,11 +152,11 @@ def data_browser_domain(self):
 
  @property
  def data_browser_name(self):
- return f'{self._resource_prefix}-data-browser-{self.deployment_stage}'
+ return f'{self.resource_prefix}-data-browser-{self.deployment_stage}'
 
  @property
  def data_portal_name(self):
- return f'{self._resource_prefix}-data-portal-{self.deployment_stage}'
+ return f'{self.resource_prefix}-data-portal-{self.deployment_stage}'
 
  @property
  def dss_endpoint(self) -> str:
@@ -308,7 +311,7 @@ def _parse_principals(self, accounts) -> MutableMapping[str, List[str]]:
  return result
 
  @property
- def _resource_prefix(self):
+ def resource_prefix(self):
  prefix = os.environ['AZUL_RESOURCE_PREFIX']
  self.validate_prefix(prefix)
  return prefix
@@ -317,50 +320,48 @@ def qualified_resource_name(self, resource_name, suffix='', stage=None):
  self._validate_term(resource_name)
  if stage is None:
  stage = self.deployment_stage
- return f"{self._resource_prefix}-{resource_name}-{stage}{suffix}"
+ return f"{self.resource_prefix}-{resource_name}-{stage}{suffix}"
 
- def unqualified_resource_name(self, qualified_resource_name: str, suffix: str = '') -> tuple:
+ def unqualified_resource_name(self,
+ qualified_resource_name: str,
+ suffix: str = ''
+ ) -> Tuple[str, str]:
  """
  >>> config.unqualified_resource_name('azul-foo-dev')
  ('foo', 'dev')
 
  >>> config.unqualified_resource_name('azul-foo')
  Traceback (most recent call last):
  ...
- azul.RequirementError
+ azul.RequirementError: ['azul', 'foo']
 
  >>> config.unqualified_resource_name('azul-object_versions-dev')
  ('object_versions', 'dev')
 
  >>> config.unqualified_resource_name('azul-object-versions-dev')
  Traceback (most recent call last):
  ...
- azul.RequirementError
-
- :param qualified_resource_name:
- :param suffix:
- :return:
+ azul.RequirementError: ['azul', 'object', 'versions', 'dev']
  """
  require(qualified_resource_name.endswith(suffix))
  if len(suffix) > 0:
  qualified_resource_name = qualified_resource_name[:-len(suffix)]
  components = qualified_resource_name.split('-')
- require(len(components) == 3)
+ require(len(components) == 3, components)
  prefix, resource_name, deployment_stage = components
- require(prefix == self._resource_prefix)
+ require(prefix == self.resource_prefix)
  return resource_name, deployment_stage
 
- def unqualified_resource_name_or_none(self, qualified_resource_name: str, suffix: str = '') -> tuple:
+ def unqualified_resource_name_or_none(self,
+ qualified_resource_name: str,
+ suffix: str = ''
+ ) -> Tuple[Optional[str], Optional[str]]:
  """
  >>> config.unqualified_resource_name_or_none('azul-foo-dev')
  ('foo', 'dev')
 
  >>> config.unqualified_resource_name_or_none('invalid-foo-dev')
  (None, None)
-
- :param qualified_resource_name:
- :param suffix:
- :return:
  """
  try:
  return self.unqualified_resource_name(qualified_resource_name, suffix=suffix)
@@ -808,6 +809,10 @@ def dynamo_object_version_table_name(self) -> str:
 
  terms_aggregation_size = 99999
 
+ @property
+ def tracked_terraform_schema(self) -> Path:
+ return Path(self.project_root) / 'terraform' / '_schema.json'
+
 
 config: Config = Config() # yes, the type hint does help PyCharm