Unisender Go: Fix status tracking webhook and tests.

- Fix signature checking to avoid false validation errors
  on webhook payloads including `/` (including all "clicked"
  and most "opened" events). And in general, avoid depending
  on specific details of Unisender Go's JSON serialization.
  (Fixes #398.)
- Handle "use single event" webhook option (which has a different
  payload format).
- Verify basic auth when Anymail's WEBHOOK_SECRET is used.
  (This is optional for Unisender Go, since payloads are signed,
  but it needs to be checked when enabled.)
- Treat "soft_bounced" events as "deferred" rather than "bounced",
  since they will be retried later.
- Update validation error to reference Project ID if the webhook
  is configured for a specific project.
- Expose Unisender Go's delivery_status code and unsubscribe form
  comment as Anymail's normalized event.description.
- Update webhook tests based on actual payloads and add several
  missing tests.
- Update docs to clarify webhook use with Unisender Go projects.

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
  - id: check-toml
  - id: check-yaml
  - id: end-of-file-fixer
+ exclude: "\\.(bin|raw)$"
  - id: fix-byte-order-marker
  - id: fix-encoding-pragma
  args: [--remove]

CHANGELOG.rst

@@ -36,6 +36,18 @@ Breaking changes
 
 * Require **Django 4.0 or later** and Python 3.8 or later.
 
+Fixes
+~~~~~
+
+* **Unisender Go:** Fix several problems in Anymail's Unisender Go status tracking
+ webhook. Rework signature checking to fix false validation errors (particularly
+ on "clicked" and "opened" events). Properly handle "use single event" webhook
+ option. Correctly verify WEBHOOK_SECRET when set. Provide Unisender Go's
+ ``delivery_status`` code and unsubscribe form ``comment`` in Anymail's
+ ``event.description``. Treat soft bounces as "deferred" rather than "bounced".
+ (Thanks to `@MikeVL`_ for fixing the signature validation problem.)
+
+
 Features
 ~~~~~~~~
 
@@ -1741,6 +1753,7 @@ Features
 .. _@mark-mishyn: https:/mark-mishyn
 .. _@martinezleoml: https:/martinezleoml
 .. _@mbk-ok: https:/mbk-ok
+.. _@MikeVL: https:/MikeVL
 .. _@mounirmesselmeni: https:/mounirmesselmeni
 .. _@mwheels: https:/mwheels
 .. _@nuschk: https:/nuschk

anymail/webhooks/unisender_go.py

@@ -8,21 +8,23 @@
 from django.http import HttpRequest, HttpResponse
 from django.utils.crypto import constant_time_compare
 
-from anymail.exceptions import AnymailWebhookValidationFailure
-from anymail.signals import AnymailTrackingEvent, EventType, RejectReason, tracking
-from anymail.utils import get_anymail_setting
-from anymail.webhooks.base import AnymailCoreWebhookView
+from ..exceptions import AnymailWebhookValidationFailure
+from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking
+from ..utils import get_anymail_setting
+from .base import AnymailBaseWebhookView
 
 
-class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
- """Handler for UniSender delivery and engagement tracking webhooks"""
+class UnisenderGoTrackingWebhookView(AnymailBaseWebhookView):
+ """Handler for Unisender Go delivery and engagement tracking webhooks"""
 
  # See https://godocs.unisender.ru/web-api-ref#callback-format for webhook payload
 
  esp_name = "Unisender Go"
  signal = tracking
  warn_if_no_basic_auth = False # because we validate against signature
 
+ api_key: str | None = None # allows kwargs override
+
  event_types = {
  "sent": EventType.SENT,
  "delivered": EventType.DELIVERED,
@@ -31,14 +33,14 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
  "unsubscribed": EventType.UNSUBSCRIBED,
  "subscribed": EventType.SUBSCRIBED,
  "spam": EventType.COMPLAINED,
- "soft_bounced": EventType.BOUNCED,
+ "soft_bounced": EventType.DEFERRED,
  "hard_bounced": EventType.BOUNCED,
  }
 
  reject_reasons = {
  "err_user_unknown": RejectReason.BOUNCED,
  "err_user_inactive": RejectReason.BOUNCED,
- "err_will_retry": RejectReason.BOUNCED,
+ "err_will_retry": None, # not rejected
  "err_mailbox_discarded": RejectReason.BOUNCED,
  "err_mailbox_full": RejectReason.BOUNCED,
  "err_spam_rejected": RejectReason.SPAM,
@@ -56,49 +58,105 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
 
  http_method_names = ["post", "head", "options", "get"]
 
+ def __init__(self, **kwargs):
+ api_key = get_anymail_setting(
+ "api_key", esp_name=self.esp_name, allow_bare=True, kwargs=kwargs
+ )
+ self.api_key_bytes = api_key.encode("ascii")
+ super().__init__(**kwargs)
+
  def get(
  self, request: HttpRequest, *args: typing.Any, **kwargs: typing.Any
  ) -> HttpResponse:
  # Unisender Go verifies the webhook with a GET request at configuration time
  return HttpResponse()
 
+ def parse_json_body(self, request: HttpRequest) -> dict | list | None:
+ # Cache parsed JSON request.body on the request.
+ if hasattr(request, "_parsed_json"):
+ parsed = getattr(request, "_parsed_json")
+ else:
+ parsed = json.loads(request.body.decode())
+ setattr(request, "_parsed_json", parsed)
+ return parsed
+
  def validate_request(self, request: HttpRequest) -> None:
  """
- How Unisender GO authenticate:
- Hash the whole request body text and replace api key in "auth" field by this hash.
-
- So it is both auth and encryption. Also, they hash JSON without spaces.
+ Validate Unisender Go webhook signature:
+ "MD5 hash of the string body of the message, with the auth value replaced
+ by the api_key of the user/project whose handler is being called."
+ https://godocs.unisender.ru/web-api-ref#callback-format
  """
- request_json = json.loads(request.body.decode("utf-8"))
- request_auth = request_json.get("auth", "")
- request_json["auth"] = get_anymail_setting(
- "api_key", esp_name=self.esp_name, allow_bare=True
- )
- json_with_key = json.dumps(request_json, separators=(",", ":"))
-
- expected_auth = md5(json_with_key.encode("utf-8")).hexdigest()
+ # This must avoid any assumptions about how Unisender Go serializes JSON
+ # (key order, spaces, Unicode encoding vs. \u escapes, etc.). But we do
+ # assume the "auth" field MD5 hash is unique within the serialized JSON,
+ # so that we can use string replacement to calculate the expected hash.
+ body = request.body
+ try:
+ parsed = self.parse_json_body(request)
+ actual_auth = parsed["auth"]
+ actual_auth_bytes = actual_auth.encode()
+ except (AttributeError, KeyError, ValueError):
+ raise AnymailWebhookValidationFailure(
+ "Unisender Go webhook called with invalid payload"
+ )
 
- if not constant_time_compare(request_auth, expected_auth):
+ body_to_sign = body.replace(actual_auth_bytes, self.api_key_bytes)
+ expected_auth = md5(body_to_sign).hexdigest()
+ if not constant_time_compare(actual_auth, expected_auth):
+ # If webhook has a selected project, include the project_id in the error.
+ try:
+ project_id = parsed["events_by_user"][0]["project_id"]
+ except (KeyError, IndexError):
+ project_id = parsed.get("project_id") # try "single event" payload
+ is_for_project = f" is for Project ID {project_id}" if project_id else ""
  raise AnymailWebhookValidationFailure(
  "Unisender Go webhook called with incorrect signature"
+ f" (check Anymail UNISENDER_GO_API_KEY setting{is_for_project})"
  )
 
  def parse_events(self, request: HttpRequest) -> list[AnymailTrackingEvent]:
- request_json = json.loads(request.body.decode("utf-8"))
- assert len(request_json["events_by_user"]) == 1 # per API docs
- esp_events = request_json["events_by_user"][0]["events"]
- return [
- self.esp_to_anymail_event(esp_event)
- for esp_event in esp_events
- if esp_event["event_name"] == "transactional_email_status"
- ]
-
- def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent:
- event_data = esp_event["event_data"]
+ parsed = self.parse_json_body(request)
+ # Unisender Go has two options for webhook payloads. We support both.
+ try:
+ events_by_user = parsed["events_by_user"]
+ except KeyError:
+ # "Use single event": one flat dict, combining "event_data" fields
+ # with "event_name", "user_id", "project_id", etc.
+ if parsed["event_name"] == "transactional_email_status":
+ esp_events = [parsed]
+ else:
+ esp_events = []
+ else:
+ # Not "use single event": we want the "event_data" from all events
+ # with event_name "transactional_email_status".
+ assert len(events_by_user) == 1 # "A single element array" per API docs
+ esp_events = [
+ event["event_data"]
+ for event in events_by_user[0]["events"]
+ if event["event_name"] == "transactional_email_status"
+ ]
+
+ return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+ def esp_to_anymail_event(self, event_data: dict) -> AnymailTrackingEvent:
  event_type = self.event_types.get(event_data["status"], EventType.UNKNOWN)
- timestamp = datetime.fromisoformat(event_data["event_time"])
- timestamp_utc = timestamp.replace(tzinfo=timezone.utc)
- metadata = event_data.get("metadata", {})
+
+ # Unisender Go does not provide any way to deduplicate webhook calls.
+ # (There is an "ID" HTTP header, but it has a unique value for every
+ # webhook call--including retransmissions of earlier failed calls.)
+ event_id = None
+
+ # event_time is ISO-like, without a stated time zone. (But it's UTC per docs.)
+ try:
+ timestamp = datetime.fromisoformat(event_data["event_time"]).replace(
+ tzinfo=timezone.utc
+ )
+ except KeyError:
+ timestamp = None
+
+ # Extract our message_id (see backend UNISENDER_GO_GENERATE_MESSAGE_ID).
+ metadata = event_data.get("metadata", {}).copy()
  message_id = metadata.pop("anymail_id", event_data.get("job_id"))
 
  delivery_info = event_data.get("delivery_info", {})
@@ -108,14 +166,18 @@ def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent:
  else:
  reject_reason = None
 
+ description = delivery_info.get("delivery_status") or event_data.get("comment")
+ mta_response = delivery_info.get("destination_response")
+
  return AnymailTrackingEvent(
  event_type=event_type,
- timestamp=timestamp_utc,
+ timestamp=timestamp,
  message_id=message_id,
- event_id=None,
+ event_id=event_id,
  recipient=event_data["email"],
  reject_reason=reject_reason,
- mta_response=delivery_info.get("destination_response"),
+ description=description,
+ mta_response=mta_response,
  metadata=metadata,
  click_url=event_data.get("url"),
  user_agent=delivery_info.get("user_agent"),

docs/esps/unisender_go.rst

@@ -329,19 +329,8 @@ Status tracking webhooks
 ------------------------
 
 If you are using Anymail's normalized :ref:`status tracking <event-tracking>`, add
-the url in Unisender Go's dashboard. Where to set the webhook depends on where
-you got your :setting:`UNISENDER_GO_API_KEY <ANYMAIL_UNISENDER_GO_API_KEY>`:
-
-* If you are using an account-level API key, configure the webhook
- under Settings > Webhooks (Настройки > Вебхуки).
-* If you are using a project-level API key, configure the webhook
- under Settings > Projects (Настройки > Проекты).
-
-(If you try to mix account-level and project-level API keys and webhooks,
-webhook signature validation will fail, and you'll get
-:exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors.)
-
-Enter these settings for the webhook:
+the url in Unisender Go's dashboard under Settings > Webhooks (Настройки > Вебхуки).
+Create a webhook with these settings:
 
 * **Notification Url:**
 
@@ -350,7 +339,8 @@ Enter these settings for the webhook:
  where *yoursite.example.com* is your Django site.
 
 * **Status:** set to "Active" if you have already deployed your Django project
- with Anymail installed. Otherwise set to "Inactive" and update after you deploy.
+ with Anymail installed. Otherwise set to "Inactive" and wait to activate it
+ until you deploy.
 
  (Unisender Go performs a GET request to verify the webhook URL
  when it is marked active.)
@@ -361,8 +351,9 @@ Enter these settings for the webhook:
  with a mod_deflate *input* filter---you could also use "json_post_compressed."
  Most web servers do not handle compressed input by default.)
 
-* **Events:** your choice. Anymail supports any combination of ``sent, delivered,
- soft_bounced, hard_bounced, opened, clicked, unsubscribed, subscribed, spam``.
+* **Events:** your choice. Anymail supports any combination of ``sent``,
+ ``delivered``, ``soft_bounced``, ``hard_bounced``, ``opened``, ``clicked``,
+ ``unsubscribed``, ``subscribed``, and/or ``spam``.
 
  Anymail does not support Unisender Go's ``spam_block`` events (but will ignore
  them if you accidentally include it).
@@ -395,9 +386,17 @@ Enter these settings for the webhook:
  :attr:`~anymail.signals.AnymailTrackingEvent.user_agent` or
  :attr:`~anymail.signals.AnymailTrackingEvent.click_url`.)
 
+* **Selected project:** Must match the project for your Anymail
+ :setting:`UNISENDER_GO_API_KEY <ANYMAIL_UNISENDER_GO_API_KEY>` setting,
+ if projects are enabled for your account and you are using a project level
+ API key. Leave blank if you are using your account level API key.
+
+ This affects webhook signing. If the selected project does not match your API key,
+ you'll get :exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors.
+
 Note that Unisender Go does not deliver tracking events for recipient
 addresses that are blocked at send time. You must check the message's
-:attr:`anymail_status.recipients[recipient_email].message_id <anymail.message.AnymailStatus.recipients>`
+:attr:`anymail_status.recipients[recipient_email].status <anymail.message.AnymailStatus.recipients>`
 immediately after sending to detect rejected recipients.
 
 Unisender Go implements webhook signing on the entire event payload,

tests/test_files/unisender-go-tracking-test-payload.json.raw

@@ -0,0 +1 @@
+{"auth":"b3cb4d6aef9d07095805c39e792e0542","events_by_user":[{"user_id":5960727,"project_name":"Testing","project_id":"6862471","events":[{"event_name":"transactional_email_status","event_data":{"job_id":"1sn15Z-0007Le-GtVN","email":"[email protected]","status":"clicked","event_time":"2024-09-08 19:56:41","url":"https:\/\/example.com","delivery_info":{"user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/128.0.0.0 Safari\/537.36","ip":"66.179.153.169"}}}]}]}