ICMSLST-2135 - Gov Notify

.env.example

@@ -28,5 +28,7 @@ SEND_LICENCE_TO_CHIEF=False
 
 SET_INACTIVE_APP_TYPES_ACTIVE=False
 
-# To send emails set GOV_NOTIFY_API_KEY to the development API key in the dev vault
-GOV_NOTIFY_API_KEY=
+# To send/receive emails (which are implemented using gov notify) set the following
+GOV_NOTIFY_API_KEY="" # use the development API key which can be found in the dev vault
+EMAIL_BACKEND="web.mail.backends.GovNotifyEmailBackend"
+SEND_ALL_EMAILS_TO="<your-email-address>"# Need to have registered with GOV Notify and have been invited to the ICMS project

Makefile

@@ -167,6 +167,11 @@ test: ## run tests (circleci; don't use locally as it produces a coverage report
  --maxprocesses=2 \
  --cov-fail-under 66 ${args}
 
+test_no_cov:
+ ./run-tests.sh \
+ --maxprocesses=2 ${args}
+
+
 migration_test: ## Run data migration tests
  ./run-tests.sh data_migration --create-db --numprocesses 2 ${args}
 

config/settings/base.py

@@ -130,12 +130,7 @@
 
 # Email
 GOV_NOTIFY_API_KEY = env.str("GOV_NOTIFY_API_KEY", default="")
-EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
-
-AWS_SES_ACCESS_KEY_ID = env.str("AWS_SES_ACCESS_KEY_ID", default="")
-AWS_SES_SECRET_ACCESS_KEY = env.str("AWS_SES_SECRET_ACCESS_KEY", default="")
-AWS_SES_REGION_NAME = "eu-west-1"
-AWS_SES_REGION_ENDPOINT = "email.eu-west-1.amazonaws.com"
+EMAIL_BACKEND = env.str("EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend")
 
 # Email/phone contacts
 EMAIL_FROM = env.str("ICMS_EMAIL_FROM", default="")
@@ -228,6 +223,7 @@
 CELERY_ACCEPT_CONTENT = ["application/json"]
 CELERY_RESULT_SERIALIZER = "json"
 CELERY_TASK_SERIALIZER = "json"
+CELERY_RESULT_EXTENDED = True
 
 # Django cache with Redis
 CACHES = {

config/settings/local.py

@@ -114,3 +114,4 @@
 
 # django-ratelimit
 RATELIMIT_ENABLE = False
+CELERY_TASK_ALWAYS_EAGER = False

config/settings/non_prod_base.py

@@ -6,11 +6,7 @@
 from .base import *
 
 # Email settings for all non prod environments.
-EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
-AWS_SES_ACCESS_KEY_ID = ""
-AWS_SES_SECRET_ACCESS_KEY = ""
-AWS_SES_REGION_NAME = ""
-AWS_SES_REGION_ENDPOINT = ""
+SEND_ALL_EMAILS_TO = env.list("SEND_ALL_EMAILS_TO", default=[])
 
 # Email/phone contacts
 EMAIL_FROM = env.str("ICMS_EMAIL_FROM", "[email protected]") # /PS-IGNORE

config/settings/test.py

@@ -57,3 +57,4 @@
 ICMS_V1_REPLICA_DSN = ""
 
 GOV_NOTIFY_API_KEY = "fakekey-11111111-1111-1111-1111-111111111111-22222222-2222-2222-222222222222"
+EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

docker-compose.yml

@@ -41,8 +41,6 @@ services:
  - ICMS_SILENCED_SYSTEM_CHECKS
  - ICMS_SECRET_KEY
  - ICMS_EMAIL_FROM
- - AWS_SES_ACCESS_KEY_ID
- - AWS_SES_SECRET_ACCESS_KEY
  - DJANGO_SETTINGS_MODULE
  - ELASTIC_APM_SECRET_TOKEN
  - ELASTIC_APM_ENVIRONMENT
@@ -64,6 +62,8 @@ services:
  - SEND_LICENCE_TO_CHIEF
  - SET_INACTIVE_APP_TYPES_ACTIVE
  - GOV_NOTIFY_API_KEY
+ - EMAIL_BACKEND
+ - SEND_ALL_EMAILS_TO
  # stdin_open: true
  # tty: true
  ports:
@@ -96,14 +96,14 @@ services:
  command: celery --app=config.celery:app worker --loglevel=info
  environment:
  - DJANGO_SETTINGS_MODULE
- - AWS_SES_ACCESS_KEY_ID
- - AWS_SES_SECRET_ACCESS_KEY
  - ICMS_HMRC_DOMAIN
  - ICMS_HMRC_UPDATE_LICENCE_ENDPOINT
  - HAWK_AUTH_ID
  - HAWK_AUTH_KEY
  - SEND_LICENCE_TO_CHIEF
  - GOV_NOTIFY_API_KEY
+ - EMAIL_BACKEND
+ - SEND_ALL_EMAILS_TO
  depends_on:
  - redis
  volumes:

requirements-base.txt

@@ -1,5 +1,5 @@
 boto3==1.21.29
-celery[redis]==5.2.7
+celery[redis]==5.3.1
 django-celery-results==2.4.0
 django-chunk-upload-handlers==0.0.11
 django-compressor==4.1
@@ -11,7 +11,6 @@ django-phonenumber-field==5.0.0
 django-ratelimit==3.0.1
 django-redis==5.2.0
 django-select2==7.10.1
-django-ses==3.5.0
 django-structlog==1.6.2
 Django==4.1.10
 elastic-apm==6.15.1
@@ -26,6 +25,7 @@ notifications-python-client==8.0.1
 openpyxl==3.0.7
 oracledb==1.2.0
 phonenumbers==8.12.12
+pytz==2023.3
 psycogreen==1.0.2
 psycopg2-binary==2.9.3
 pydantic==1.10.2

requirements-dev.txt

@@ -29,7 +29,7 @@ pip-check
 pre-commit==2.20.0
 boto3-stubs[essential]
 types-openpyxl==3.1.0.0
-types-pytz==2021.1.0
+types-pytz==2023.3
 types-python-dateutil==2.8.19.4
 types-Pygments==2.14.0.6
 types-requests==2.25.0

web/domains/case/access/views.py

@@ -17,6 +17,7 @@
 )
 from web.domains.case.services import case_progress, reference
 from web.flow.models import ProcessTypes
+from web.mail import emails
 from web.models import (
  AccessRequest,
  ExporterAccessRequest,
@@ -115,7 +116,7 @@ def importer_access_request(request: AuthenticatedHttpRequest) -> HttpResponse:
  process=access_request, task_type=Task.TaskType.PROCESS, owner=request.user
  )
 
- notify.send_access_requested_email(access_request)
+ emails.send_access_requested_email(access_request)
 
  if request.user.has_perm(Perms.sys.importer_access) or request.user.has_perm(
  Perms.sys.exporter_access
@@ -157,7 +158,7 @@ def exporter_access_request(request: AuthenticatedHttpRequest) -> HttpResponse:
  process=access_request, task_type=Task.TaskType.PROCESS, owner=request.user
  )
 
- notify.send_access_requested_email(access_request)
+ emails.send_access_requested_email(access_request)
 
  if request.user.has_perm(Perms.sys.importer_access) or request.user.has_perm(
  Perms.sys.exporter_access

web/mail/api.py

@@ -4,6 +4,8 @@
 from notifications_python_client import NotificationsAPIClient
 from notifications_python_client.errors import HTTPError
 
+from config.celery import app
+
 
 def get_gov_notify_client() -> NotificationsAPIClient:
  return NotificationsAPIClient(settings.GOV_NOTIFY_API_KEY)
@@ -25,3 +27,11 @@ def is_valid_template_id(template_id: UUID) -> bool:
  except ValueError:
  return False
  return gov_notify_template_id == template_id
+
+
+@app.task
+def send_email(template_id: UUID, personalisation: dict, email_address: str) -> dict:
+ client = get_gov_notify_client()
+ return client.send_email_notification(
+ email_address, str(template_id), personalisation=personalisation
+ )

web/mail/backends.py

@@ -0,0 +1,19 @@
+from uuid import UUID
+
+import structlog as logging
+from django.core.mail.backends.base import BaseEmailBackend
+
+from web.mail.api import send_email
+
+logger = logging.getLogger(__name__)
+
+
+class GovNotifyEmailBackend(BaseEmailBackend):
+ def send_messages(self, email_messages: list) -> None:
+ for message in email_messages:
+ for recipient in message.recipients():
+ logger.info(f"Sending {message.name.label} email to {recipient}")
+ self.send_message(message.template_id, message.personalisation, recipient)
+
+ def send_message(self, template_id: UUID, personalisation: dict, recipient: str) -> None:
+ send_email.delay(template_id, personalisation, recipient)

web/mail/decorators.py

@@ -0,0 +1,18 @@
+from functools import wraps
+
+from django.conf import settings
+
+
+def override_recipients(f):
+ """Helper decorator to override the email addresses returned by the wrapped function.
+ If APP_ENV is dev or local and SEND_ALL_EMAILS_TO is set in django settings all emails will be sent to
+ the specified email addresses.
+ """
+
+ @wraps(f)
+ def wrapper(*args, **kwargs):
+ if settings.APP_ENV in ("local", "dev") and settings.SEND_ALL_EMAILS_TO:
+ return settings.SEND_ALL_EMAILS_TO
+ return f(*args, **kwargs)
+
+ return wrapper

web/mail/emails.py

@@ -0,0 +1,11 @@
+from web.models import AccessRequest
+
+from .messages import AccessRequestEmail
+from .recipients import get_ilb_case_officers_email_addresses
+
+
+def send_access_requested_email(access_request: AccessRequest) -> None:
+ recipients = get_ilb_case_officers_email_addresses()
+ for recipient in recipients:
+ email = AccessRequestEmail(access_request, to=[recipient])
+ email.send()

web/mail/messages.py

@@ -0,0 +1,48 @@
+from uuid import UUID
+
+from django.conf import settings
+from django.core.mail import EmailMessage, SafeMIMEMultipart
+
+from .constants import EmailTypes
+from .models import EmailTemplate
+
+
+class GOVNotifyEmailMessage(EmailMessage):
+ name = None | EmailTypes
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.template_id = self.get_template_id()
+ self.personalisation = self.get_personalisation()
+
+ def message(self) -> SafeMIMEMultipart:
+ """Adds the personalisation data to the message header, so it is visible when using the console backend."""
+ message = super().message()
+ message["Personalisation"] = self.personalisation
+ return message
+
+ def get_context(self) -> dict:
+ raise NotImplementedError
+
+ def get_personalisation(self) -> dict:
+ return {
+ "icms_url": settings.DEFAULT_DOMAIN,
+ "icms_contact_email": settings.ILB_CONTACT_EMAIL,
+ "icms_contact_phone": settings.ILB_CONTACT_PHONE,
+ "subject": self.subject,
+ "body": self.body,
+ } | self.get_context()
+
+ def get_template_id(self) -> UUID:
+ return EmailTemplate.objects.get(name=self.name).gov_notify_template_id
+
+
+class AccessRequestEmail(GOVNotifyEmailMessage):
+ name = EmailTypes.ACCESS_REQUEST
+
+ def __init__(self, access_request, *args, **kwargs):
+ self.access_request = access_request
+ super().__init__(*args, **kwargs)
+
+ def get_context(self) -> dict:
+ return {"reference": self.access_request.reference}

web/mail/recipients.py

@@ -0,0 +1,19 @@
+from django.db.models import QuerySet
+
+from web.mail.decorators import override_recipients
+from web.models import User
+from web.notify.utils import get_notification_emails
+from web.permissions import get_ilb_case_officers
+
+
+def get_ilb_case_officers_email_addresses() -> list[str]:
+ users = get_ilb_case_officers()
+ return get_email_addresses_for_users(users)
+
+
+@override_recipients
+def get_email_addresses_for_users(users: QuerySet[User]) -> list[str]:
+ emails = []
+ for user in users:
+ emails.extend(get_notification_emails(user))
+ return list(set(emails))

web/notify/notify.py

@@ -26,7 +26,7 @@
  User,
  VariationRequest,
 )
-from web.permissions import SysPerms, get_ilb_case_officers
+from web.permissions import SysPerms
 
 from . import email, utils
 
@@ -81,15 +81,6 @@ def register(user, password):
  )
 
 
-def send_access_requested_email(access_request):
- context = {"subject": f"Access Request {access_request.reference}"}
- email.send_html_email(
- "email/access/access_requested.html",
- context,
- list(get_ilb_case_officers()),
- )
-
-
 def access_request_closed(access_request):
  requester = access_request.submitted_by
  subject = "Import Case Management System Account"

web/tests/conftest.py

@@ -1,4 +1,5 @@
 import datetime
+from unittest import mock
 
 import pytest
 from django.conf import settings
@@ -8,6 +9,7 @@
 from django.test.client import Client
 from django.urls import reverse
 from jinja2 import Template as Jinja2Template
+from notifications_python_client import NotificationsAPIClient
 from pytest_django.asserts import assertRedirects
 
 from web.domains.case.services import case_progress, document_pack
@@ -726,6 +728,22 @@ def strict_templates():
  yield None
 
 
+@pytest.fixture
+def enable_gov_notify_backend():
+ with override_settings(EMAIL_BACKEND="web.mail.backends.GovNotifyEmailBackend"):
+ yield None
+
+
+@pytest.fixture
+def mock_gov_notify_client(enable_gov_notify_backend):
+ with mock.patch("web.mail.api.get_gov_notify_client") as client:
+ mock_gov_notify_client = mock.create_autospec(
+ spec=NotificationsAPIClient(settings.GOV_NOTIFY_API_KEY), instance=True
+ )
+ client.return_value = mock_gov_notify_client
+ yield mock_gov_notify_client
+
+
 def _set_valid_licence(app):
  licence = document_pack.pack_draft_get(app)
  licence.case_completion_datetime = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)