Add webhook subscription available when creating project from REST API …

…#98 (#380)

Signed-off-by: Thomas Druez <[email protected]>

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 Unreleased
 ----------
 
+- Add webhook subscription available when creating project from REST API.
+ https:/nexB/scancode.io/issues/98
+
 - Add the project "reset" feature in the UI, CLI, and REST API.
  https:/nexB/scancode.io/issues/375
 

docs/rest-api.rst

@@ -69,6 +69,17 @@ An API endpoint that provides the ability to list, get, and create projects.
  ]
  }
 
+The project list can be filtered by ``name``, ``uuid``, and ``is_archived`` fields.
+For example:
+
+.. code-block:: console
+
+ api_url="http://localhost/api/projects/"
+ content_type="Content-Type: application/json"
+ payload="name=project_name"
+
+ curl -X GET "$api_url?$payload" -H "$content_type"
+
 Create a project
 ----------------
 

scanpipe/api/serializers.py

@@ -107,6 +107,7 @@ class ProjectSerializer(
  required=False,
  style={"base_template": "textarea.html"},
  )
+ webhook_url = serializers.CharField(write_only=True, required=False)
  next_run = serializers.CharField(source="get_next_run", read_only=True)
  runs = RunSerializer(many=True, read_only=True)
  input_sources = serializers.SerializerMethodField()
@@ -121,6 +122,7 @@ class Meta:
  "uuid",
  "upload_file",
  "input_urls",
+ "webhook_url",
  "created_date",
  "is_archived",
  "pipeline",
@@ -177,6 +179,7 @@ def create(self, validated_data):
  input_urls = validated_data.pop("input_urls", [])
  pipeline = validated_data.pop("pipeline", None)
  execute_now = validated_data.pop("execute_now", False)
+ webhook_url = validated_data.pop("webhook_url", None)
 
  downloads, errors = fetch_urls(input_urls)
  if errors:
@@ -193,6 +196,9 @@ def create(self, validated_data):
  if pipeline:
  project.add_pipeline(pipeline, execute_now)
 
+ if webhook_url:
+ project.add_webhook_subscription(webhook_url)
+
  return project
 
 

scanpipe/api/views.py

@@ -69,6 +69,7 @@ class ProjectViewSet(
 
  queryset = Project.objects.all()
  serializer_class = ProjectSerializer
+ filterset_fields = ["name", "uuid", "is_archived"]
 
  @action(detail=True, renderer_classes=[renderers.JSONRenderer])
  def results(self, request, *args, **kwargs):

scanpipe/migrations/0014_webhooksubscription.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.9 on 2021-12-07 12:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('scanpipe', '0013_project_is_archived'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='WebhookSubscription',
+ fields=[
+ ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+ ('target_url', models.URLField(max_length=1024, verbose_name='Target URL')),
+ ('sent', models.BooleanField(default=False)),
+ ('created_date', models.DateTimeField(auto_now_add=True)),
+ ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='webhooksubscriptions', to='scanpipe.project')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ ]

scanpipe/models.py

@@ -21,6 +21,7 @@
 # Visit https:/nexB/scancode.io for support and download.
 
 import inspect
+import json
 import logging
 import re
 import shutil
@@ -35,6 +36,7 @@
 from django.conf import settings
 from django.core import checks
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.db import transaction
 from django.db.models import Q
@@ -50,6 +52,7 @@
 
 import django_rq
 import redis
+import requests
 from packageurl import normalize_qualifiers
 from packageurl.contrib.django.models import PackageURLQuerySetMixin
 from rq.command import send_stop_job_command
@@ -745,6 +748,13 @@ def add_pipeline(self, pipeline_name, execute_now=False):
  transaction.on_commit(run.execute_task_async)
  return run
 
+ def add_webhook_subscription(self, target_url):
+ """
+ Creates a new WebhookSubscription instance with the provided `target_url` for
+ the current project.
+ """
+ return WebhookSubscription.objects.create(project=self, target_url=target_url)
+
  def get_next_run(self):
  """
  Returns the next non-executed Run instance assigned to current project.
@@ -1151,6 +1161,13 @@ def append_to_log(self, message, save=False):
  if save:
  self.save()
 
+ def send_project_subscriptions(self):
+ """
+ Triggers related project webhook subscriptions.
+ """
+ for subscription in self.project.webhooksubscriptions.all():
+ subscription.send(pipeline_run=self)
+
  def profile(self, print_results=False):
  """
  Returns computed execution times for each step in the current Run.
@@ -1752,3 +1769,49 @@ def create_from_data(cls, project, package_data):
  # can be injected in the ProjectError record.
  discovered_package.save(save_error=False, capture_exception=False)
  return discovered_package
+
+
+class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
+ target_url = models.URLField(_("Target URL"), max_length=1024)
+ sent = models.BooleanField(default=False)
+ created_date = models.DateTimeField(auto_now_add=True, editable=False)
+
+ def __str__(self):
+ return str(self.uuid)
+
+ def send(self, pipeline_run):
+ """
+ Sends this WebhookSubscription by POSTing an HTTP request on the `target_url`.
+ """
+ payload = {
+ "project": {
+ "uuid": self.project.uuid,
+ "name": self.project.name,
+ "input_sources": self.project.input_sources,
+ },
+ "run": {
+ "uuid": pipeline_run.uuid,
+ "pipeline_name": pipeline_run.pipeline_name,
+ "status": pipeline_run.status,
+ "scancodeio_version": pipeline_run.scancodeio_version,
+ },
+ }
+
+ logger.info(f"Sending Webhook uuid={self.uuid}.")
+ try:
+ response = requests.post(
+ url=self.target_url,
+ data=json.dumps(payload, cls=DjangoJSONEncoder),
+ headers={"Content-Type": "application/json"},
+ timeout=10,
+ )
+ except requests.exceptions.RequestException as exception:
+ logger.info(exception)
+ return
+
+ if response.status_code in (200, 201, 202):
+ logger.info(f"Webhook uuid={self.uuid} sent and received.")
+ self.sent = True
+ self.save()
+ else:
+ logger.info(f"Webhook uuid={self.uuid} returned a {response.status_code}.")

scanpipe/tasks.py

@@ -65,6 +65,7 @@ def execute_pipeline_task(run_pk):
 
  info("Update Run instance with exitcode, output, and end_date", run_pk)
  run.set_task_ended(exitcode, output, refresh_first=True)
+ run.send_project_subscriptions()
 
  if run.task_succeeded:
  # We keep the temporary files available for debugging in case of error

scanpipe/tests/test_api.py

@@ -86,6 +86,32 @@ def test_scanpipe_api_project_list(self):
  self.assertNotContains(response, "resource_count")
  self.assertNotContains(response, "package_count")
 
+ def test_scanpipe_api_project_list_filters(self):
+ project2 = Project.objects.create(name="project2", is_archived=True)
+
+ response = self.csrf_client.get(self.project_list_url)
+ self.assertEqual(2, response.data["count"])
+ self.assertContains(response, self.project1.uuid)
+ self.assertContains(response, project2.uuid)
+
+ data = {"uuid": self.project1.uuid}
+ response = self.csrf_client.get(self.project_list_url, data=data)
+ self.assertEqual(1, response.data["count"])
+ self.assertContains(response, self.project1.uuid)
+ self.assertNotContains(response, project2.uuid)
+
+ data = {"name": project2.name}
+ response = self.csrf_client.get(self.project_list_url, data=data)
+ self.assertEqual(1, response.data["count"])
+ self.assertNotContains(response, self.project1.uuid)
+ self.assertContains(response, project2.uuid)
+
+ data = {"is_archived": True}
+ response = self.csrf_client.get(self.project_list_url, data=data)
+ self.assertEqual(1, response.data["count"])
+ self.assertNotContains(response, self.project1.uuid)
+ self.assertContains(response, project2.uuid)
+
  def test_scanpipe_api_project_detail(self):
  response = self.csrf_client.get(self.project1_detail_url)
  self.assertIn(self.project1_detail_url, response.data["url"])

scanpipe/tests/test_models.py

@@ -49,6 +49,7 @@
 from scanpipe.models import ProjectError
 from scanpipe.models import Run
 from scanpipe.models import RunInProgressError
+from scanpipe.models import WebhookSubscription
 from scanpipe.models import get_project_work_directory
 from scanpipe.pipes.fetch import Download
 from scanpipe.pipes.input import copy_input
@@ -344,6 +345,11 @@ def test_scanpipe_project_model_add_uploads(self):
  self.assertEqual(expected, inputs)
  self.assertEqual({}, missing_inputs)
 
+ def test_scanpipe_project_model_add_webhook_subscription(self):
+ self.assertEqual(0, self.project1.webhooksubscriptions.count())
+ self.project1.add_webhook_subscription("https://localhost")
+ self.assertEqual(1, self.project1.webhooksubscriptions.count())
+
  def test_scanpipe_project_model_get_next_run(self):
  self.assertEqual(None, self.project1.get_next_run())
 
@@ -669,6 +675,13 @@ def test_scanpipe_run_model_append_to_log(self):
  run1.refresh_from_db()
  self.assertEqual("line1\nline2\n", run1.log)
 
+ @mock.patch("scanpipe.models.WebhookSubscription.send")
+ def test_scanpipe_run_model_send_project_subscriptions(self, mock_send):
+ self.project1.add_webhook_subscription("https://localhost")
+ run1 = self.create_run()
+ run1.send_project_subscriptions()
+ mock_send.assert_called_once_with(pipeline_run=run1)
+
  def test_scanpipe_run_model_profile_method(self):
  run1 = self.create_run()
  self.assertIsNone(run1.profile())
@@ -1170,6 +1183,22 @@ def test_scanpipe_codebase_resource_model_walk_method(self):
  ]
  self.assertEqual(expected_bottom_up_paths, bottom_up_paths)
 
+ @mock.patch("requests.post")
+ def test_scanpipe_webhook_subscription_send_method(self, mock_post):
+ webhook = self.project1.add_webhook_subscription("https://localhost")
+ self.assertFalse(webhook.sent)
+ run1 = self.create_run()
+
+ mock_post.return_value = mock.Mock(status_code=404)
+ webhook.send(pipeline_run=run1)
+ webhook.refresh_from_db()
+ self.assertFalse(webhook.sent)
+
+ mock_post.return_value = mock.Mock(status_code=200)
+ webhook.send(pipeline_run=run1)
+ webhook.refresh_from_db()
+ self.assertTrue(webhook.sent)
+
 
 class ScanPipeModelsTransactionTest(TransactionTestCase):
  """