fix: include licenses for bundled lz4_flex and Zstandard

.github/workflows/ci.yml

@@ -175,6 +175,9 @@ jobs:
       - name: Apache RAT license check
         run: ./dev/release/check-rat.sh HEAD
 
+      - name: Test JAR contents checker
+        run: python3 dev/ci/test-jar-contents.py
+
       - name: Setup Node.js
         uses: actions/setup-node@v7
         with:

dev/ci/test-jar-contents.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Check source-only JAR entries against both artifact roles using real ZIP files."""
+
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[2]
+CHECKER = ROOT / "dev/ensure-jars-have-correct-contents.sh"
+LICENSE = (ROOT / "dev/licenses/LICENSE-binary.txt").read_bytes()
+
+
+class JarContents(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.main = self.jar("main.jar")
+
+    def jar(self, name, entry=None):
+        path = self.root / name
+        with zipfile.ZipFile(path, "w") as archive:
+            archive.writestr("org/apache/comet/linux/amd64/libcomet.so", b"")
+            archive.writestr("META-INF/LICENSE", LICENSE)
+            if entry:
+                archive.writestr(entry, "// source\n")
+        return str(path)
+
+    def check(self, *args):
+        return subprocess.run(
+            ["bash", str(CHECKER), *args], capture_output=True, text=True)
+
+    def rejected(self, args, entry):
+        result = self.check(*args)
+        self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertIn("unexpected contents", result.stdout)
+        self.assertIn(entry, result.stdout)
+
+    def check_source_entry(self, entry):
+        artifact = self.jar("payload.jar", entry)
+        self.rejected([artifact], entry)
+        result = self.check(self.main, artifact)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        # Permissions must not leak into the main list, even for the same path.
+        self.rejected([self.main + ";" + artifact, artifact], entry)
+        self.rejected([artifact, artifact], entry)
+
+    def test_module_descriptor(self):
+        self.check_source_entry("module-info.java")
+
+    def test_plugins_source(self):
+        self.check_source_entry("org/apache/spark/Plugins.scala")
+
+    def test_row_partition_source(self):
+        self.check_source_entry("org/apache/spark/shuffle/sort/RowPartition.scala")
+
+    def test_unrelated_source_rejected_in_both_roles(self):
+        entry = "Unexpected.java"
+        artifact = self.jar("unexpected.jar", entry)
+        self.rejected([artifact], entry)
+        self.rejected([self.main, artifact], entry)
+
+
+if __name__ == "__main__":
+    unittest.main()

dev/ensure-jars-have-correct-contents.sh

@@ -18,12 +18,11 @@
 
 # Borrowed from Hadoop
 
-# Usage: $0 [/path/to/some/example.jar;/path/to/another/example/created.jar]
+# Usage: $0 'main.jar;other.jar' [sources.jar] [skip-sources=false]
 #
-# accepts a single command line argument with a colon separated list of
-# paths to jars to check. Iterates through each such passed jar and checks
-# all the contained paths to make sure they follow the below constructed
-# safe list.
+# The first argument is a semicolon-separated list of JARs to check. An optional
+# sources JAR is also checked unless the third argument is true (maven.source.skip).
+# Checks all contained paths against the safe list below.
 
 # We use +=, which is a bash 3.1+ feature
 if [[ -z "${BASH_VERSINFO[0]}" ]] \
@@ -36,6 +35,42 @@ fi
 set -e
 set -o pipefail
 
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+binary_license="${script_dir}/licenses/LICENSE-binary.txt"
+
+locked_version ()
+{
+  # Cargo generates these package/name/version records. Reject absent or ambiguous versions.
+  awk -v wanted="$1" '
+    /^\[\[package\]\]/ { name = "" }
+    $1 == "name" && $2 == "=" { name = $3; gsub(/["\r]/, "", name) }
+    $1 == "version" && $2 == "=" && name == wanted {
+      version = $3; gsub(/["\r]/, "", version); count++
+    }
+    END { if (count != 1) exit 1; print version }
+  ' "${script_dir}/../native/Cargo.lock"
+}
+
+check_native_license_versions ()
+{
+  local lz4_version zstd_sys_version zstd_version
+  if ! lz4_version=$(locked_version lz4_flex) || ! zstd_sys_version=$(locked_version zstd-sys); then
+    echo "[ERROR] Expected one lz4_flex and one zstd-sys version in native/Cargo.lock."
+    return 1
+  fi
+  case "${zstd_sys_version}" in
+    *+zstd.*) zstd_version=${zstd_sys_version##*+zstd.} ;;
+    *) echo "[ERROR] Cannot read the vendored Zstandard version from zstd-sys ${zstd_sys_version}."; return 1 ;;
+  esac
+  if ! tr -d '\r' < "${binary_license}" |
+      grep -Fx "The native Comet library bundles lz4_flex ${lz4_version} (MIT)." > /dev/null ||
+      ! tr -d '\r' < "${binary_license}" |
+      grep -Fx "The native Comet library bundles Zstandard ${zstd_version} (BSD-3-Clause)." > /dev/null; then
+    echo "[ERROR] Update LICENSE-binary.txt for locked lz4_flex ${lz4_version} / Zstandard ${zstd_version}."
+    return 1
+  fi
+}
+
 allowed_expr="(^org/$|^org/apache/$"
 # we have to allow the directories that lead to the org/apache/comet dir
 # We allow all the classes under the following packages:
@@ -114,6 +149,12 @@ allowed_expr+="|^scala/collection/"
 allowed_expr+="|^scala/jdk/"
 allowed_expr+="|^scala/util/"
 
+# Source-only exceptions apply to the optional sources artifact, not the main list.
+sources_allowed_expr="${allowed_expr}"
+sources_allowed_expr+="|^org/apache/spark/Plugins\.scala$"
+sources_allowed_expr+="|^org/apache/spark/shuffle/sort/RowPartition\.scala$"
+sources_allowed_expr+="|^module-info\.java$"
+sources_allowed_expr+=")"
 allowed_expr+=")"
 declare -i bad_artifacts=0
 declare -a bad_contents
@@ -124,17 +165,37 @@ if [ "${#artifact_list[@]}" -eq 0 ]; then
   exit 1
 fi
 
+sources_index=-1
+if [ -n "${2:-}" ]; then
+  skip_sources=$(printf '%s' "${3:-false}" | tr '[:upper:]' '[:lower:]')
+  if [ "${skip_sources}" = "true" ]; then
+    echo "[INFO] Skipping sources-JAR verification (maven.source.skip=true)."
+  elif [ -f "$2" ]; then
+    sources_index=${#artifact_list[@]}
+    artifact_list+=("$2")
+  else
+    echo "[ERROR] Expected sources JAR does not exist: '$2'."
+    echo "[ERROR] Override comet.spark.sources.jar for a custom path, or set maven.source.skip=true for an intentional skip."
+    exit 1
+  fi
+fi
+
 jar_list_failed ()
 {
     echo "[ERROR] Listing jar contents for file '${artifact}' failed."
     exit 1
 }
 trap jar_list_failed SIGUSR1
 
-for artifact in "${artifact_list[@]}"; do
+for artifact_index in "${!artifact_list[@]}"; do
+  artifact="${artifact_list[artifact_index]}"
+  artifact_allowed_expr="${allowed_expr}"
+  if [ "${artifact_index}" -eq "${sources_index}" ]; then
+    artifact_allowed_expr="${sources_allowed_expr}"
+  fi
   bad_contents=()
   # Note: On Windows the output from jar tf may contain \r\n's.  Normalize to \n.
-  while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${allowed_expr}" )
+  while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${artifact_allowed_expr}" )
   if [ ${#bad_contents[@]} -gt 0 ]; then
     echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
     echo "    Please check the following and either correct the build or update"
@@ -147,6 +208,26 @@ for artifact in "${artifact_list[@]}"; do
   else
     echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'"
   fi
+
+  # Check the complete license text in both main and sources jars containing libcomet.
+  # Common/test-only jars without the native library keep their existing legal resources.
+  if jar tf "${artifact}" | tr -d '\r' |
+      grep -E '^org/apache/comet/[^/]+/[^/]+/(libcomet\.(so|dylib)|comet\.dll)$' > /dev/null; then
+    if ! check_native_license_versions; then
+      bad_artifacts=${bad_artifacts}+1
+      continue
+    fi
+    license_count=$(jar tf "${artifact}" | tr -d '\r' | awk '$0 == "META-INF/LICENSE" { n++ } END { print n+0 }')
+    if [ "${license_count}" -ne 1 ]; then
+      echo "[ERROR] Expected exactly one META-INF/LICENSE in '${artifact}'."
+      bad_artifacts=${bad_artifacts}+1
+    elif ! unzip -p "${artifact}" META-INF/LICENSE | cmp -s "${binary_license}" -; then
+      echo "[ERROR] Binary LICENSE differs or is missing native attribution in '${artifact}'."
+      bad_artifacts=${bad_artifacts}+1
+    else
+      echo "[INFO] Binary LICENSE matches: '$(basename "${artifact}")'"
+    fi
+  fi
 done
 
 if [ "${bad_artifacts}" -gt 0 ]; then