Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for unknown licenses in attribution output #749 #750

Merged
merged 1 commit into from
May 30, 2023
Merged

Add support for unknown licenses in attribution output #749 #750

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 35 additions & 6 deletions scanpipe/pipes/output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from licensedcode.cache import build_spdx_license_expression
from licensedcode.cache import get_licenses_by_spdx_key
from licensedcode.cache import get_licensing
from licensedcode.models import License
from scancode_config import __version__ as scancode_toolkit_version

from scancodeio import SCAN_NOTICE
Expand Down Expand Up @@ -635,9 +636,7 @@ def to_cyclonedx(project):


def get_expression_as_attribution_links(parsed_expression):
template = (
'<a href="#license_{symbol.wrapped.key}">{symbol.wrapped.spdx_license_key}</a>'
)
template = '<a href="#license_{symbol.key}">{symbol.wrapped.spdx_license_key}</a>'
return parsed_expression.simplify().render(template=template)


Expand All @@ -660,6 +659,38 @@ def get_attribution_template(project):
return default_template


def make_unknown_license_object(license_symbol):
"""
Return a ``License`` object suitable for the provided ``license_symbol``,
that is representing a license key unknown by the current toolkit licensed index.
"""
mocked_spdx_license_key = f"LicenseRef-unknown-{license_symbol.key}"
return License(
key=license_symbol.key,
spdx_license_key=mocked_spdx_license_key,
text="ERROR: Unknown license key, no text available.",
is_builtin=False,
)


def get_package_expression_symbols(parsed_expression):
"""
Return the list of ``license_symbols`` contained in the ``parsed_expression``.
Since unknown license keys are missing a ``License`` set in the ``wrapped``
attribute, a special "unknown" ``License`` object is injected.
"""
license_symbols = []

for parsed_symbol in parsed_expression.symbols:
# .decompose() is required for LicenseWithExceptionSymbol support
for license_symbol in parsed_symbol.decompose():
if not hasattr(license_symbol, "wrapped"):
license_symbol.wrapped = make_unknown_license_object(license_symbol)
license_symbols.append(license_symbol)

return license_symbols


def to_attribution(project):
"""
Generate attribution for the provided ``project``.
Expand All @@ -678,10 +709,8 @@ def to_attribution(project):
for package in packages:
if package.declared_license_expression:
parsed = licensing.parse(package.declared_license_expression)
license_symbols.extend(get_package_expression_symbols(parsed))
package.expression_links = get_expression_as_attribution_links(parsed)
# .decompose() is required for LicenseWithExceptionSymbol support
for symbol in parsed.symbols:
license_symbols.extend(list(symbol.decompose()))

licenses = [symbol.wrapped for symbol in set(license_symbols)]
licenses.sort(key=attrgetter("spdx_license_key"))
Expand Down
8 changes: 5 additions & 3 deletions scanpipe/templates/scanpipe/attribution.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,11 @@ <h2 id="license_texts">Licenses that apply to {{ project.name }}</h2>
<h3 id="license_{{ license.key }}">
{{ license.spdx_license_key }}
</h3>
<a href="{{ license.licensedb_url}}" target="_blank">
{{ license.licensedb_url}}
</a>
{% if license.licensedb_url %}
<a href="{{ license.licensedb_url}}" target="_blank">
{{ license.licensedb_url}}
</a>
{% endif %}
<pre>{{ license.text }}</pre>
<hr>
{% endfor %}
Expand Down
35 changes: 25 additions & 10 deletions scanpipe/tests/data/outputs/expected_attribution.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ <h2>The following licenses are used in Analysis:</h2>

<li><a href="#license_gpl-2.0">GPL-2.0-only</a></li>

<li><a href="#license_missing-unknown">LicenseRef-unknown-missing-unknown</a></li>

<li><a href="#license_mit">MIT</a></li>

</ul>
Expand Down Expand Up @@ -89,7 +91,7 @@ <h3>pkg:deb/debian/[email protected]?arch=all</h3>



<p>This package is licensed under <a href="#license_gpl-2.0">GPL-2.0-only</a> WITH <a href="#license_classpath-exception-2.0">Classpath-exception-2.0</a> AND <a href="#license_mit">MIT</a></p>
<p>This package is licensed under <a href="#license_gpl-2.0">GPL-2.0-only</a> WITH <a href="#license_classpath-exception-2.0">Classpath-exception-2.0</a> AND <a href="#license_missing-unknown">LicenseRef-unknown-missing-unknown</a> AND <a href="#license_mit">MIT</a></p>



Expand All @@ -107,9 +109,11 @@ <h2 id="license_texts">Licenses that apply to Analysis</h2>
<h3 id="license_classpath-exception-2.0">
Classpath-exception-2.0
</h3>
<a href="https://scancode-licensedb.aboutcode.org/classpath-exception-2.0" target="_blank">
https://scancode-licensedb.aboutcode.org/classpath-exception-2.0
</a>

<a href="https://scancode-licensedb.aboutcode.org/classpath-exception-2.0" target="_blank">
https://scancode-licensedb.aboutcode.org/classpath-exception-2.0
</a>

<pre>Linking this library statically or dynamically with other modules is making a
combined work based on this library. Thus, the terms and conditions of the GNU
General Public License cover the whole combination.
Expand All @@ -129,9 +133,11 @@ <h3 id="license_classpath-exception-2.0">
<h3 id="license_gpl-2.0">
GPL-2.0-only
</h3>
<a href="https://scancode-licensedb.aboutcode.org/gpl-2.0" target="_blank">
https://scancode-licensedb.aboutcode.org/gpl-2.0
</a>

<a href="https://scancode-licensedb.aboutcode.org/gpl-2.0" target="_blank">
https://scancode-licensedb.aboutcode.org/gpl-2.0
</a>

<pre> GNU GENERAL PUBLIC LICENSE
Version 2, June 1991

Expand Down Expand Up @@ -473,12 +479,21 @@ <h3 id="license_gpl-2.0">
Public License instead of this License.</pre>
<hr>

<h3 id="license_missing-unknown">
LicenseRef-unknown-missing-unknown
</h3>

<pre>ERROR: Unknown license key, no text available.</pre>
<hr>

<h3 id="license_mit">
MIT
</h3>
<a href="https://scancode-licensedb.aboutcode.org/mit" target="_blank">
https://scancode-licensedb.aboutcode.org/mit
</a>

<a href="https://scancode-licensedb.aboutcode.org/mit" target="_blank">
https://scancode-licensedb.aboutcode.org/mit
</a>

<pre>Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
&quot;Software&quot;), to deal in the Software without restriction, including
Expand Down
33 changes: 29 additions & 4 deletions scanpipe/tests/pipes/test_output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,35 @@ def test_scanpipe_pipes_outputs_to_spdx(self):
output_file = output.to_spdx(project=project)
self.assertIn(output_file.name, project.output_root)

def test_scanpipe_pipes_outputs_make_unknown_license_object(self):
licensing = get_licensing()
parsed_expression = licensing.parse("some-unknown-license")

self.assertEqual(1, len(parsed_expression.symbols))
license_symbol = list(parsed_expression.symbols)[0]
license_object = output.make_unknown_license_object(license_symbol)

self.assertEqual("some-unknown-license", license_object.key)
self.assertEqual(
"LicenseRef-unknown-some-unknown-license", license_object.spdx_license_key
)
self.assertEqual(
"ERROR: Unknown license key, no text available.", license_object.text
)
self.assertFalse(license_object.is_builtin)

def test_scanpipe_pipes_outputs_get_package_expression_symbols(self):
licensing = get_licensing()
parsed_expression = licensing.parse("mit AND some-unknown-license")
symbols = output.get_package_expression_symbols(parsed_expression)
self.assertEqual(2, len(symbols))
self.assertTrue(hasattr(symbols[0], "wrapped"))
self.assertTrue(hasattr(symbols[1], "wrapped"))

def test_scanpipe_pipes_outputs_get_expression_as_attribution_links(self):
expression_with_exception = "mit AND gpl-2.0 with classpath-exception-2.0"
expression = "mit AND gpl-2.0 with classpath-exception-2.0"
licensing = get_licensing()
parsed_expression = licensing.parse(expression_with_exception)
parsed_expression = licensing.parse(expression)
rendered = output.get_expression_as_attribution_links(parsed_expression)
expected = (
'<a href="#license_gpl-2.0">GPL-2.0-only</a>'
Expand Down Expand Up @@ -305,8 +330,8 @@ def test_scanpipe_pipes_outputs_get_attribution_template(self):
def test_scanpipe_pipes_outputs_to_attribution(self):
project = Project.objects.create(name="Analysis")
package_data = dict(package_data1)
expression_with_exception = "mit AND gpl-2.0 with classpath-exception-2.0"
package_data["declared_license_expression"] = expression_with_exception
expression = "mit AND gpl-2.0 with classpath-exception-2.0 AND missing-unknown"
package_data["declared_license_expression"] = expression
package_data["notice_text"] = "Notice text"
pipes.update_or_create_package(project, package_data)

Expand Down