Add compact JSON output for scancode

docs/source/reference/scancode-cli/cli-help-text-options.rst

@@ -402,7 +402,7 @@ for ScanCode Version 32.0.0.
 
 
   --------------------------------------------
-  Plugin: scancode_output:json  class: formattedcode.output_json:JsonCompactOutput
+  Plugin: scancode_output:json  class: formattedcode.output_json:JsonOutput
     codebase_attributes:
     resource_attributes:
     sort_order: 100

pyproject-scancode-toolkit-mini.toml

@@ -306,7 +306,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights"
 [project.entry-points.scancode_output]
 html = "formattedcode.output_html:HtmlOutput"
 html-app = "formattedcode.output_html:HtmlAppOutput"
-json = "formattedcode.output_json:JsonCompactOutput"
+json = "formattedcode.output_json:JsonOutput"
+compact-json = "formattedcode.output_json:JsonCompactOutput"
 json-pp = "formattedcode.output_json:JsonPrettyOutput"
 spdx-tv = "formattedcode.output_spdx:SpdxTvOutput"
 spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput"

pyproject-scancode-toolkit.toml

@@ -307,7 +307,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights"
 [project.entry-points.scancode_output]
 html = "formattedcode.output_html:HtmlOutput"
 html-app = "formattedcode.output_html:HtmlAppOutput"
-json = "formattedcode.output_json:JsonCompactOutput"
+json = "formattedcode.output_json:JsonOutput"
+compact-json = "formattedcode.output_json:JsonCompactOutput"
 json-pp = "formattedcode.output_json:JsonPrettyOutput"
 spdx-tv = "formattedcode.output_spdx:SpdxTvOutput"
 spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput"

pyproject.toml

@@ -311,7 +311,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights"
 [project.entry-points.scancode_output]
 html = "formattedcode.output_html:HtmlOutput"
 html-app = "formattedcode.output_html:HtmlAppOutput"
-json = "formattedcode.output_json:JsonCompactOutput"
+json = "formattedcode.output_json:JsonOutput"
+compact-json = "formattedcode.output_json:JsonCompactOutput"
 json-pp = "formattedcode.output_json:JsonPrettyOutput"
 spdx-tv = "formattedcode.output_spdx:SpdxTvOutput"
 spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput"

src/formattedcode/output_compact.py

@@ -0,0 +1,233 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+"""
+Helper functions to write scan results in a compact manner as JSON, preserving
+only non-duplicated key detection information, and skipping attributes with
+large values or more details.
+"""
+
+# Tracing flags
+TRACE = False
+
+
+def logger_debug(*args):
+    pass
+
+
+if TRACE:
+    import sys
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str)
+                                     and a or repr(a) for a in args))
+
+
+COMPACT_CONFLICTING_OPTIONS = [
+    'summary', 'classify', 'consolidate', 'todo', 'mark_source', 'tallies',
+    'tallies_by_facet', 'tallies_with_details', 'tallies_key_files',
+    'license_references', 'license_text', 'license_diagnostics', 'license_text_diagnostics',
+    'go_symbol', 'rust_symbol'
+]
+
+PACKAGE_DETECTION_FIELDS = [
+    # Detection fields have a lot of inlined duplicated data which needs to be skipped,
+    # just identifiers are enough, and this will be kept in the data mapping seperately
+    "license_detections",
+    "other_license_detections"
+]
+
+COMPACT_PACKAGES_SKIP_ATTRIBUTES = [
+    # These are PackageURL fields and are coontained in `package_uid` which is a purl + identifier
+    "type",
+    "namespace",
+    "name",
+    "version",
+    "qualifiers",
+    "subpath",
+    "purl",
+    # This usually has text which is too large and not useful for a compact output 
+    "description",
+    # Just `declared_license_expression_spdx` is enough
+    "declared_license_expression",
+    # Just `other_license_expression_spdx` is enough
+    "other_license_expression_spdx",
+    # Already captured by license detection fields if present
+    "extracted_license_statement",
+    # This usually has text which is too large and not useful for a compact output 
+    "notice_text",
+    "extra_data",  
+    # This can be deduced from datafile_paths
+    "datasource_ids"
+] + PACKAGE_DETECTION_FIELDS
+
+
+COMPACT_DEPENDENCIES_SKIP_ATTRIBUTES = [
+    "resolved_package",
+    # Not essential information
+    "extra_data",
+    # This can be deduced from `datafile_path`
+    "datasource_id"
+]
+
+COMPACT_LICENSES_SKIP_ATTRIBUTES = [
+    # Just keeping `license_expression_spdx` is enough
+    "license_expression",
+    # This needs to be compact further, so we can't have as-is 
+    "reference_matches"
+]
+
+COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES = [
+    # Just keeping `license_expression_spdx` is enough
+    "license_expression",
+    # We have the `rule_identifier` and this is derivative
+    "rule_url",
+    # We have the `rule_identifier` and this info can be fetched 
+    "rule_relevance"
+]
+
+COMPACT_RESOURCES_SKIP_ATTRIBUTES = [
+    # Just keeping `detected_license_expression_spdx` is enough
+    "detected_license_expression",
+    # Detection fields have a lot of inlined duplicated data which needs to be skipped,
+    # just identifiers are enough, and this will be kept in the data mapping seperately
+    "license_detections",
+    "package_data",
+    # license clues are matches not aggregated in the top-level license detections, but
+    # these mappings need to be processed to make them more compact
+    "license_clues",
+    # `name`, `base_name` and `extension` info is also present in the full-path
+    "name",
+    "base_name",
+    "extension"
+]
+
+def get_compact_packages(package_mapping):
+    compact_package_mapping = get_compact_mapping(
+        mapping=package_mapping,
+        keys_to_skip=COMPACT_PACKAGES_SKIP_ATTRIBUTES,
+    )
+
+    # Only keep the license detection identifiers from the detection mappings
+    for detection_attribute in PACKAGE_DETECTION_FIELDS:
+        detection_mapping = [
+            {"identifier": detection.get("identifier")}
+            for detection in package_mapping.get(detection_attribute, [])
+            if detection
+        ]
+        if detection_mapping:
+            compact_package_mapping[detection_attribute] = detection_mapping
+
+    return compact_package_mapping
+
+
+def get_compact_dependencies(dependency_mapping):
+    return get_compact_mapping(
+        mapping=dependency_mapping,
+        keys_to_skip=COMPACT_DEPENDENCIES_SKIP_ATTRIBUTES,
+    )
+
+
+def get_compact_license_detections(license_mapping):
+    compact_license_mapping = get_compact_mapping(
+        mapping=license_mapping,
+        keys_to_skip=COMPACT_LICENSES_SKIP_ATTRIBUTES,
+    )
+    compact_license_mapping["reference_matches"] = [
+        get_compact_mapping(
+            mapping=match,
+            keys_to_skip=COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES,
+        )
+        for match in license_mapping.get("reference_matches", [])
+    ]
+    return compact_license_mapping
+
+
+def get_compact_mapping(mapping, keys_to_skip=[]):
+    """
+    Get a more compact version of `mapping` which doesn't have
+    empty key/value pairs or attributes which are explicitly skipped
+    with the `keys_to_skip` list.
+
+    Also removes empty key/value pairs recursively.
+    """
+    if type(mapping) is not dict:
+        return mapping
+
+    return {
+        key: get_compact_mapping(value)
+        for key, value in mapping.items()
+        if value and key not in keys_to_skip
+    }
+
+
+get_compact_func_by_key = {
+    "packages": get_compact_packages,
+    "dependencies": get_compact_dependencies,
+    "license_detections": get_compact_license_detections,
+}
+
+def write_compact_codebase_attributes(stream, key, value):
+    if not key in get_compact_func_by_key:
+        return
+
+    func = get_compact_func_by_key.get(key)
+    if func:
+        compact_output = [
+            func(instance)
+            for instance in value
+        ]
+        if compact_output:
+            return stream.write(key, compact_output)
+
+
+def get_compact_resource(resource_mapping):
+    compact_resource_mapping = get_compact_mapping(
+        mapping=resource_mapping,
+        keys_to_skip=COMPACT_RESOURCES_SKIP_ATTRIBUTES,
+    )
+
+    # Only keep detection identifiers for resource level package/license
+    # detections as these are assembled into top-level summaries and so
+    # these inlined data is duplicate information
+    package_data = [
+        {"purl": detection.get("purl")}
+        for detection in resource_mapping.get("package_data", [])
+        if detection and detection.get("purl")
+    ]
+    if package_data:
+        compact_resource_mapping["package_data"] = package_data
+
+    license_detections = [
+        {"identifier": detection.get("identifier")}
+        for detection in resource_mapping.get("license_detections", [])
+        if detection
+    ]
+    if license_detections:
+        compact_resource_mapping["license_detections"] = license_detections
+
+    # license clues are matches not aggregated in the top-level license detections, but
+    # these mappings need to be processed to make them more compact
+    license_clues = [
+        get_compact_mapping(
+            mapping=clue_mapping,
+            keys_to_skip=COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES,
+        )
+        for clue_mapping in resource_mapping.get("license_clues", [])
+        if clue_mapping
+    ]
+    if license_clues:
+        compact_resource_mapping["license_clues"] = license_clues
+
+    return compact_resource_mapping

src/formattedcode/output_json.py

@@ -9,6 +9,9 @@
 import jsonstreams
 
 from formattedcode import FileOptionType
+from formattedcode.output_compact import COMPACT_CONFLICTING_OPTIONS
+from formattedcode.output_compact import write_compact_codebase_attributes
+from formattedcode.output_compact import get_compact_resource
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import OUTPUT_GROUP
 from plugincode.output import output_impl
@@ -40,14 +43,14 @@ def logger_debug(*args):
 
 
 @output_impl
-class JsonCompactOutput(OutputPlugin):
+class JsonOutput(OutputPlugin):
 
     options = [
         PluggableCommandLineOption(('--json', 'output_json',),
             type=FileOptionType(mode='w', encoding='utf-8', lazy=True),
             metavar='FILE',
             default=None,
-            help='Write scan output as compact JSON to FILE.',
+            help='Write scan output as JSON to FILE.',
             help_group=OUTPUT_GROUP,
             sort_order=10),
     ]
@@ -59,6 +62,27 @@ def process_codebase(self, codebase, output_json, **kwargs):
         write_results(codebase, output_file=output_json, pretty=False, **kwargs)
 
 
+@output_impl
+class JsonCompactOutput(OutputPlugin):
+
+    options = [
+        PluggableCommandLineOption(('--compact-json', 'output_compact_json',),
+            type=FileOptionType(mode='w', encoding='utf-8', lazy=True),
+            metavar='FILE',
+            default=None,
+            conflicting_options=COMPACT_CONFLICTING_OPTIONS,
+            help='Write scan output as compact JSON to FILE.',
+            help_group=OUTPUT_GROUP,
+            sort_order=10),
+    ]
+
+    def is_enabled(self, output_compact_json, **kwargs):
+        return output_compact_json
+
+    def process_codebase(self, codebase, output_compact_json, **kwargs):
+        write_results(codebase, output_file=output_compact_json, pretty=False, compact=True, **kwargs)
+
+
 @output_impl
 class JsonPrettyOutput(OutputPlugin):
 
@@ -79,11 +103,12 @@ def process_codebase(self, codebase, output_json_pp, **kwargs):
         write_results(codebase, output_file=output_json_pp, pretty=True, **kwargs)
 
 
-def write_results(codebase, output_file, pretty=False, **kwargs):
+def write_results(codebase, output_file, pretty=False, compact=False, **kwargs):
     """
     Write headers, files, and other attributes from `codebase` to `output_file`
 
     Enable JSON indentation if `pretty` is True
+    Only return useful, non-duplicated 
     """
     # Set indentation for JSON output if `pretty` is True
     # We use a separate dict for jsonstream kwargs since we are passing
@@ -114,12 +139,22 @@ def write_results(codebase, output_file, pretty=False, **kwargs):
         # Write attributes
         if codebase.attributes:
             for attribute_key, attribute_value in codebase.attributes.to_dict().items():
-                s.write(attribute_key, attribute_value)
+                if compact:
+                    write_compact_codebase_attributes(stream=s, key=attribute_key, value=attribute_value)
+                else:
+                    s.write(attribute_key, attribute_value)
 
         # Write files
         codebase_files = OutputPlugin.get_files(codebase, **kwargs)
         # OutputPlugin.get_files() returns a generator, not JSON-serializable
-        codebase_files = list(codebase_files)
+        if compact:
+            codebase_files = list(
+                get_compact_resource(codebase_file)
+                for codebase_file in codebase_files
+            )
+        else:
+            codebase_files = list(codebase_files)
+
         s.write('files', codebase_files)