diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fc66aba7f4..b8008d7390d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,9 @@ jobs: - name: Apache RAT license check run: ./dev/release/check-rat.sh HEAD + - name: Test JAR contents checker + run: python3 dev/ci/test-jar-contents.py + - name: Setup Node.js uses: actions/setup-node@v7 with: diff --git a/dev/ci/test-jar-contents.py b/dev/ci/test-jar-contents.py new file mode 100644 index 00000000000..32127233173 --- /dev/null +++ b/dev/ci/test-jar-contents.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check source-only JAR entries against both artifact roles using real ZIP files.""" + +from pathlib import Path +import subprocess +import tempfile +import unittest +import zipfile + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "dev/ensure-jars-have-correct-contents.sh" +LICENSE = (ROOT / "dev/licenses/LICENSE-binary.txt").read_bytes() + + +class JarContents(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.main = self.jar("main.jar") + + def jar(self, name, entry=None): + path = self.root / name + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("org/apache/comet/linux/amd64/libcomet.so", b"") + archive.writestr("META-INF/LICENSE", LICENSE) + if entry: + archive.writestr(entry, "// source\n") + return str(path) + + def check(self, *args): + return subprocess.run( + ["bash", str(CHECKER), *args], capture_output=True, text=True) + + def rejected(self, args, entry): + result = self.check(*args) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("unexpected contents", result.stdout) + self.assertIn(entry, result.stdout) + + def check_source_entry(self, entry): + artifact = self.jar("payload.jar", entry) + self.rejected([artifact], entry) + result = self.check(self.main, artifact) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + # Permissions must not leak into the main list, even for the same path. + self.rejected([self.main + ";" + artifact, artifact], entry) + self.rejected([artifact, artifact], entry) + + def test_module_descriptor(self): + self.check_source_entry("module-info.java") + + def test_plugins_source(self): + self.check_source_entry("org/apache/spark/Plugins.scala") + + def test_row_partition_source(self): + self.check_source_entry("org/apache/spark/shuffle/sort/RowPartition.scala") + + def test_unrelated_source_rejected_in_both_roles(self): + entry = "Unexpected.java" + artifact = self.jar("unexpected.jar", entry) + self.rejected([artifact], entry) + self.rejected([self.main, artifact], entry) + + +if __name__ == "__main__": + unittest.main() diff --git a/dev/ensure-jars-have-correct-contents.sh b/dev/ensure-jars-have-correct-contents.sh index 1907743667b..15b398dd94c 100755 --- a/dev/ensure-jars-have-correct-contents.sh +++ b/dev/ensure-jars-have-correct-contents.sh @@ -18,12 +18,11 @@ # Borrowed from Hadoop -# Usage: $0 [/path/to/some/example.jar;/path/to/another/example/created.jar] +# Usage: $0 'main.jar;other.jar' [sources.jar] [skip-sources=false] # -# accepts a single command line argument with a colon separated list of -# paths to jars to check. Iterates through each such passed jar and checks -# all the contained paths to make sure they follow the below constructed -# safe list. +# The first argument is a semicolon-separated list of JARs to check. An optional +# sources JAR is also checked unless the third argument is true (maven.source.skip). +# Checks all contained paths against the safe list below. # We use +=, which is a bash 3.1+ feature if [[ -z "${BASH_VERSINFO[0]}" ]] \ @@ -36,6 +35,42 @@ fi set -e set -o pipefail +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +binary_license="${script_dir}/licenses/LICENSE-binary.txt" + +locked_version () +{ + # Cargo generates these package/name/version records. Reject absent or ambiguous versions. + awk -v wanted="$1" ' + /^\[\[package\]\]/ { name = "" } + $1 == "name" && $2 == "=" { name = $3; gsub(/["\r]/, "", name) } + $1 == "version" && $2 == "=" && name == wanted { + version = $3; gsub(/["\r]/, "", version); count++ + } + END { if (count != 1) exit 1; print version } + ' "${script_dir}/../native/Cargo.lock" +} + +check_native_license_versions () +{ + local lz4_version zstd_sys_version zstd_version + if ! lz4_version=$(locked_version lz4_flex) || ! zstd_sys_version=$(locked_version zstd-sys); then + echo "[ERROR] Expected one lz4_flex and one zstd-sys version in native/Cargo.lock." + return 1 + fi + case "${zstd_sys_version}" in + *+zstd.*) zstd_version=${zstd_sys_version##*+zstd.} ;; + *) echo "[ERROR] Cannot read the vendored Zstandard version from zstd-sys ${zstd_sys_version}."; return 1 ;; + esac + if ! tr -d '\r' < "${binary_license}" | + grep -Fx "The native Comet library bundles lz4_flex ${lz4_version} (MIT)." > /dev/null || + ! tr -d '\r' < "${binary_license}" | + grep -Fx "The native Comet library bundles Zstandard ${zstd_version} (BSD-3-Clause)." > /dev/null; then + echo "[ERROR] Update LICENSE-binary.txt for locked lz4_flex ${lz4_version} / Zstandard ${zstd_version}." + return 1 + fi +} + allowed_expr="(^org/$|^org/apache/$" # we have to allow the directories that lead to the org/apache/comet dir # We allow all the classes under the following packages: @@ -114,6 +149,12 @@ allowed_expr+="|^scala/collection/" allowed_expr+="|^scala/jdk/" allowed_expr+="|^scala/util/" +# Source-only exceptions apply to the optional sources artifact, not the main list. +sources_allowed_expr="${allowed_expr}" +sources_allowed_expr+="|^org/apache/spark/Plugins\.scala$" +sources_allowed_expr+="|^org/apache/spark/shuffle/sort/RowPartition\.scala$" +sources_allowed_expr+="|^module-info\.java$" +sources_allowed_expr+=")" allowed_expr+=")" declare -i bad_artifacts=0 declare -a bad_contents @@ -124,6 +165,21 @@ if [ "${#artifact_list[@]}" -eq 0 ]; then exit 1 fi +sources_index=-1 +if [ -n "${2:-}" ]; then + skip_sources=$(printf '%s' "${3:-false}" | tr '[:upper:]' '[:lower:]') + if [ "${skip_sources}" = "true" ]; then + echo "[INFO] Skipping sources-JAR verification (maven.source.skip=true)." + elif [ -f "$2" ]; then + sources_index=${#artifact_list[@]} + artifact_list+=("$2") + else + echo "[ERROR] Expected sources JAR does not exist: '$2'." + echo "[ERROR] Override comet.spark.sources.jar for a custom path, or set maven.source.skip=true for an intentional skip." + exit 1 + fi +fi + jar_list_failed () { echo "[ERROR] Listing jar contents for file '${artifact}' failed." @@ -131,10 +187,15 @@ jar_list_failed () } trap jar_list_failed SIGUSR1 -for artifact in "${artifact_list[@]}"; do +for artifact_index in "${!artifact_list[@]}"; do + artifact="${artifact_list[artifact_index]}" + artifact_allowed_expr="${allowed_expr}" + if [ "${artifact_index}" -eq "${sources_index}" ]; then + artifact_allowed_expr="${sources_allowed_expr}" + fi bad_contents=() # Note: On Windows the output from jar tf may contain \r\n's. Normalize to \n. - while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${allowed_expr}" ) + while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${artifact_allowed_expr}" ) if [ ${#bad_contents[@]} -gt 0 ]; then echo "[ERROR] Found artifact with unexpected contents: '${artifact}'" echo " Please check the following and either correct the build or update" @@ -147,6 +208,26 @@ for artifact in "${artifact_list[@]}"; do else echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'" fi + + # Check the complete license text in both main and sources jars containing libcomet. + # Common/test-only jars without the native library keep their existing legal resources. + if jar tf "${artifact}" | tr -d '\r' | + grep -E '^org/apache/comet/[^/]+/[^/]+/(libcomet\.(so|dylib)|comet\.dll)$' > /dev/null; then + if ! check_native_license_versions; then + bad_artifacts=${bad_artifacts}+1 + continue + fi + license_count=$(jar tf "${artifact}" | tr -d '\r' | awk '$0 == "META-INF/LICENSE" { n++ } END { print n+0 }') + if [ "${license_count}" -ne 1 ]; then + echo "[ERROR] Expected exactly one META-INF/LICENSE in '${artifact}'." + bad_artifacts=${bad_artifacts}+1 + elif ! unzip -p "${artifact}" META-INF/LICENSE | cmp -s "${binary_license}" -; then + echo "[ERROR] Binary LICENSE differs or is missing native attribution in '${artifact}'." + bad_artifacts=${bad_artifacts}+1 + else + echo "[INFO] Binary LICENSE matches: '$(basename "${artifact}")'" + fi + fi done if [ "${bad_artifacts}" -gt 0 ]; then diff --git a/dev/licenses/LICENSE-binary.txt b/dev/licenses/LICENSE-binary.txt new file mode 100644 index 00000000000..15a1d83db25 --- /dev/null +++ b/dev/licenses/LICENSE-binary.txt @@ -0,0 +1,264 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +The native Comet library bundles lz4_flex 0.14.0 (MIT). +https://github.com/PSeitz/lz4_flex/blob/1bffdcbbf906a234b913937cb2f57c0245915038/LICENSE + +The MIT License (MIT) + +Copyright (c) 2020 Pascal Seitz + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The native Comet library bundles Zstandard 1.5.7 (BSD-3-Clause). +https://github.com/facebook/zstd/blob/v1.5.7/LICENSE + +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3347c2a8ac8..b9b0b5e9802 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -6,6 +6,8 @@ .claude/skills CLAUDE.md native/Cargo.lock +# Binary license contains the Apache license and unmodified upstream license texts. +dev/licenses/LICENSE-binary.txt dev/copyright/scala-header.txt dev/diffs/iceberg/*.diff dev/release/requirements.txt diff --git a/pom.xml b/pom.xml index a215e2a818f..161fa8d69e3 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,8 @@ under the License. UTF-8 UTF-8 + + false 17 + + + + + ${project.basedir}/../spark/target/comet-spark-spark${spark.version.short}_${scala.binary.version}-${project.version}-sources.jar + @@ -101,6 +106,8 @@ under the License. ${project.basedir}/../dev/ensure-jars-have-correct-contents.sh ${comet-artifacts} + ${comet.spark.sources.jar} + ${maven.source.skip} diff --git a/spark/pom.xml b/spark/pom.xml index 16a43db3200..4b217d3a334 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -800,8 +800,9 @@ under the License. separately and writes the name twice, failing the build. Excluding the already relocated copy on the way in leaves exactly one. --> META-INF/services/${comet.shade.packageName}.arrow.* - + + META-INF/LICENSE META-INF/NOTICE @@ -856,6 +857,11 @@ under the License. and its contents. --> + + META-INF/LICENSE + ${project.basedir}/../dev/licenses/LICENSE-binary.txt +