Merge branch 'main' into resolve-dependencies-from-lockfile

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

.gitignore

@@ -47,7 +47,6 @@ local
 /.python-version
 /.pytest_cache/
 /scancodeio.egg-info/
-policies.yml
 *.rdb
 *.aof
 .vscode

CHANGELOG.rst

@@ -1,7 +1,21 @@
 Changelog
 =========
 
-v34.6.3 (unreleased)
+v34.6.4 (unreleased)
+--------------------
+
+- Add all "classify" plugin fields from scancode-toolkit on the CodebaseResource model.
+ https:/nexB/scancode.io/issues/1275
+
+- Refine the extraction errors reporting to include the resource path for rendering
+ link to the related resources in the UI.
+ https:/nexB/scancode.io/issues/1273
+
+- Add a ``flush-projects`` management command, to Delete all project data and their
+ related work directories created more than a specified number of days ago.
+ https:/nexB/scancode.io/issues/1289
+
+v34.6.3 (2024-06-21)
 --------------------
 
 - Use the ``--option=value`` syntax for args entries in place of ``--option value``
@@ -11,6 +25,9 @@ v34.6.3 (unreleased)
 - Fix an issue in the d2d JavaScript mapper.
  https:/nexB/scancode.io/pull/1274
 
+- Add support for a ``ignored_vulnerabilities`` field on the Project configuration.
+ https:/nexB/scancode.io/issues/1271
+
 v34.6.2 (2024-06-18)
 --------------------
 

docs/command-line-interface.rst

@@ -298,6 +298,27 @@ Optional arguments:
 - ``--no-input`` Does not prompt the user for input of any kind.
 
 
+.. _cli_flush_projects:
+
+`$ scanpipe flush-projects`
+---------------------------
+
+Delete all project data and their related work directories created more than a
+specified number of days ago.
+
+Optional arguments:
+
+- ``---retain-days RETAIN_DAYS`` Specify the number of days to retain data.
+ All data older than this number of days will be deleted.
+ **Defaults to 0 (delete all data)**.
+
+ For example, to delete all projects created more than one week ago::
+
+ scanpipe flush-projects --retain-days 7
+
+- ``--no-input`` Does not prompt the user for input of any kind.
+
+
 .. _cli_create_user:
 
 `$ scanpipe create-user <username>`

docs/faq.rst

@@ -249,3 +249,32 @@ Note that only the HTTPS type of URL is supported::
 A GitHub repository URL example::
 
  https:/username/repository.git
+
+How can I cleanup my ScanCode.io installation, removing all projects and related data?
+--------------------------------------------------------------------------------------
+
+You can use the :ref:`cli_flush_projects` command to perform bulk deletion of projects
+and their associated data stored on disk::
+
+ $ scanpipe flush-projects
+
+**Confirmation will be required before deletion.**
+
+To automate this process, such as running it from a cron job, you can use the
+``--no-input`` option to skip confirmation prompts.
+
+Additionally, you can retain specific projects and their data based on their
+creation date using the ``--retain-days`` option.
+
+Here's an example of a crontab entry that runs daily and flushes all projects and
+data older than 7 days::
+
+ @daily scanpipe flush-projects --retain-days 7 --no-input
+
+.. note:: If you are use Docker for running ScanCode.io, you can run the scanpipe
+ ``flush-projects`` command using::
+
+ docker compose run --rm web scanpipe flush-projects
+
+ See :ref:`command_line_interface` chapter for more information about the scanpipe
+ command.

docs/project-configuration.rst

@@ -52,16 +52,20 @@ Content of a ``scancode-config.yml`` file:
  product_name: My Product Name
  product_version: '1.0'
  ignored_patterns:
-  - '*.tmp'
-  - 'tests/*'
+ - '*.tmp'
+ - 'tests/*'
  ignored_dependency_scopes:
  - package_type: npm
  scope: devDependencies
  - package_type: pypi
  scope: tests
+ ignored_vulnerabilities:
+ - VCID-q4q6-yfng-aaag
+ - CVE-2024-27351
+ - GHSA-vm8q-m57g-pff3
 
-See the :ref:`project_configuration_settings` section for the details about each
-setting.
+See the following :ref:`project_configuration_settings` section for the details about
+each setting.
 
 .. tip::
  You can generate the project configuration file from the
@@ -125,10 +129,10 @@ packages, define the following in your ``scancode-config.yml`` configuration fil
 .. code-block:: yaml
 
  ignored_dependency_scopes:
-  - package_type: npm
-  scope: devDependencies
-  - package_type: pypi
-  scope: tests
+ - package_type: npm
+ scope: devDependencies
+ - package_type: pypi
+ scope: tests
 
 If you prefer to use the :ref:`user_interface_project_settings` form, list each
 ignored scope using the `package_type:scope` syntax, **one per line**, such as:
@@ -141,3 +145,21 @@ ignored scope using the `package_type:scope` syntax, **one per line**, such as:
 .. warning::
  Be precise when listing scope names to avoid unintended exclusions.
  Ensure the scope names are correct and reflect your project requirements.
+
+ignored_vulnerabilities
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Provide one or more vulnerability id to be ignored, **one per line**.
+
+You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or
+``GHSA``.
+
+.. code-block:: yaml
+
+ ignored_vulnerabilities:
+ - VCID-q4q6-yfng-aaag
+ - CVE-2024-27351
+ - GHSA-vm8q-m57g-pff3
+ - OSV-2020-871
+ - BIT-django-2024-24680
+ - PYSEC-2024-28

docs/scancode-config.yml

@@ -9,12 +9,18 @@
 # - Use ``*`` to match multiple characters.
 # - Use ``?`` to match a single character.
 #ignored_patterns:
-#  - '*.tmp'
-#  - 'tests/*'
+# - '*.tmp'
+# - 'tests/*'
 
 # Specify certain dependency scopes to be ignored for a given package type.
 #ignored_dependency_scopes:
 # - package_type: npm
 # scope: devDependencies
 # - package_type: pypi
 # scope: tests
+
+# Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases.
+#ignored_vulnerabilities:
+# - VCID-q4q6-yfng-aaag
+# - CVE-2024-27351
+# - GHSA-vm8q-m57g-pff3

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.25.1
-download_url: https:/pypa/get-virtualenv/raw/20.25.1/public/virtualenv.pyz
+version: 20.26.3
+download_url: https:/pypa/get-virtualenv/raw/20.26.3/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https:/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.25.1#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.26.3#public/virtualenv.pyz

scancodeio/__init__.py

@@ -28,7 +28,7 @@
 
 import git
 
-VERSION = "34.6.2"
+VERSION = "34.6.3"
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent

scanpipe/api/serializers.py

@@ -315,6 +315,10 @@ class Meta:
  "is_text",
  "is_archive",
  "is_media",
+ "is_legal",
+ "is_manifest",
+ "is_readme",
+ "is_top_level",
  "is_key_file",
  "detected_license_expression",
  "detected_license_expression_spdx",

scanpipe/filters.py

@@ -566,8 +566,12 @@ class Meta:
  "is_binary",
  "is_text",
  "is_archive",
- "is_key_file",
  "is_media",
+ "is_legal",
+ "is_manifest",
+ "is_readme",
+ "is_top_level",
+ "is_key_file",
  ]
 
  def __init__(self, *args, **kwargs):

scanpipe/forms.py

@@ -365,10 +365,16 @@ def prepare_value(self, value):
 """
 
 
+ignored_vulnerabilities_help = """
+Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases.
+"""
+
+
 class ProjectSettingsForm(forms.ModelForm):
  settings_fields = [
  "ignored_patterns",
  "ignored_dependency_scopes",
+ "ignored_vulnerabilities",
  "attribution_template",
  "product_name",
  "product_version",
@@ -399,6 +405,18 @@ class ProjectSettingsForm(forms.ModelForm):
  key_name="package_type",
  value_name="scope",
  )
+ ignored_vulnerabilities = ListTextarea(
+ label="Ignored vulnerabilities",
+ required=False,
+ help_text=convert_markdown_to_html(ignored_vulnerabilities_help.strip()),
+ widget=forms.Textarea(
+ attrs={
+ "class": "textarea is-dynamic",
+ "rows": 2,
+ "placeholder": "VCID-q4q6-yfng-aaag\nCVE-2024-27351",
+ },
+ ),
+ )
  attribution_template = forms.CharField(
  label="Attribution template",
  required=False,

scanpipe/management/commands/flush-projects.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https:/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https:/nexB/scancode.io for support and download.
+
+import datetime
+import sys
+
+from django.core.management.base import BaseCommand
+from django.template.defaultfilters import pluralize
+from django.utils import timezone
+
+from scanpipe.models import Project
+
+
+class Command(BaseCommand):
+ help = (
+ "Delete all project data and their related work directories created more than "
+ "a specified number of days ago."
+ )
+
+ def add_arguments(self, parser):
+ super().add_arguments(parser)
+ parser.add_argument(
+ "--retain-days",
+ type=int,
+ help=(
+ "Optional. Specify the number of days to retain data. "
+ "All data older than this number of days will be deleted. "
+ "Defaults to 0 (delete all data)."
+ ),
+ default=0,
+ )
+ parser.add_argument(
+ "--no-input",
+ action="store_false",
+ dest="interactive",
+ help="Do not prompt the user for input of any kind.",
+ )
+
+ def handle(self, *inputs, **options):
+ verbosity = options["verbosity"]
+ retain_days = options["retain_days"]
+ projects = Project.objects.all()
+
+ if retain_days:
+ cutoff_date = timezone.now() - datetime.timedelta(days=retain_days)
+ projects = projects.filter(created_date__lt=cutoff_date)
+
+ projects_count = projects.count()
+ if projects_count == 0:
+ if verbosity > 0:
+ self.stdout.write("No projects to remove.")
+ sys.exit(0)
+
+ if options["interactive"]:
+ confirm = input(
+ f"You have requested the deletion of {projects_count} "
+ f"project{pluralize(projects_count)}.\n"
+ "This will IRREVERSIBLY DESTROY all data related to those projects.\n"
+ "Are you sure you want to do this?\n"
+ "Type 'yes' to continue, or 'no' to cancel: "
+ )
+ if confirm != "yes":
+ if verbosity > 0:
+ self.stdout.write("Flush cancelled.")
+ sys.exit(0)
+
+ deletion_count = 0
+ for project in projects:
+ project.delete()
+ deletion_count += 1
+
+ if verbosity > 0:
+ msg = (
+ f"{deletion_count} project{pluralize(deletion_count)} and "
+ f"{pluralize(deletion_count, 'its,their')} related data have been "
+ f"removed."
+ )
+ self.stdout.write(msg, self.style.SUCCESS)