diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml index 9c2c74593fa6f..17bd794263582 100644 --- a/.github/actions/setup-gradle/action.yml +++ b/.github/actions/setup-gradle/action.yml @@ -18,7 +18,7 @@ # name: Setup Gradle -description: Sets up Gradle with Develocity or public build scan publishing by default +description: Sets up Gradle with runner memory defaults and build scans for public repositories inputs: develocity-access-key: description: 'Develocity access key for authenticated build scans' @@ -28,6 +28,10 @@ inputs: description: 'Whether to publish build scans or use Develocity when the access key is set' required: false default: 'true' + memory-profile: + description: 'Gradle memory profile: auto (low-memory on Linux runners with up to 8 GiB RAM), low-memory, or standard' + required: false + default: 'auto' cache-read-only: description: 'Whether the Gradle cache is read-only' required: false @@ -40,7 +44,7 @@ runs: using: composite steps: - name: Set Develocity Project ID and configure custom settings - if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }} shell: bash run: | mkdir -p ~/.gradle @@ -49,23 +53,29 @@ runs: grep -q 'systemProp.scan.uploadInBackground=' ~/.gradle/gradle.properties || echo systemProp.scan.uploadInBackground=false >> ~/.gradle/gradle.properties - name: Setup Gradle with Develocity - if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }} + if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }} uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: develocity-injection-enabled: true develocity-url: https://develocity.apache.org # expected format is develocity.apache.org: develocity-access-key: ${{ inputs.develocity-access-key }} - build-scan-publish: ${{ inputs.build-scan-publish }} + build-scan-publish: ${{ inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }} cache-read-only: ${{ inputs.cache-read-only }} add-job-summary: ${{ inputs.add-job-summary }} - name: Setup Gradle - if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true') }} + if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public') }} uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: - build-scan-publish: ${{ inputs.build-scan-publish }} + build-scan-publish: ${{ inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }} build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service' build-scan-terms-of-use-agree: 'yes' cache-read-only: ${{ inputs.cache-read-only }} - add-job-summary: ${{ inputs.add-job-summary }} \ No newline at end of file + add-job-summary: ${{ inputs.add-job-summary }} + + - name: Configure Gradle memory + shell: bash + env: + MEMORY_PROFILE: ${{ inputs.memory-profile }} + run: bash "$GITHUB_ACTION_PATH/configure-memory.sh" "$MEMORY_PROFILE" diff --git a/.github/actions/setup-gradle/configure-memory.sh b/.github/actions/setup-gradle/configure-memory.sh new file mode 100644 index 0000000000000..f71dc15b336b1 --- /dev/null +++ b/.github/actions/setup-gradle/configure-memory.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +profile="${1:-auto}" +case "$profile" in + auto) + profile=standard + if [[ -r /proc/meminfo ]]; then + memory_kib=$(awk '/^MemTotal:/ { print $2 }' /proc/meminfo) + if [[ "$memory_kib" =~ ^[0-9]+$ ]] && (( memory_kib > 0 && memory_kib <= 8 * 1024 * 1024 )); then + profile=low-memory + fi + fi + ;; + low-memory|standard) ;; + *) echo "Unknown Gradle memory profile: $profile" >&2; exit 1 ;; +esac + +echo "Gradle memory profile: $profile" +if [[ "$profile" == low-memory ]]; then + gradle_dir="${GRADLE_USER_HOME:-$HOME/.gradle}" + mkdir -p "$gradle_dir" + # Bound workers across projects as well as forks within each test task. Leave + # room for native JVM memory and containers, and recycle accumulated test state. + printf '\n' >> "$gradle_dir/gradle.properties" + cat >> "$gradle_dir/gradle.properties" <<'EOF' +org.gradle.jvmargs=-Xmx2g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp +org.gradle.workers.max=2 +testMaxParallelForks=2 +testForkEvery=50 +EOF +fi diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 7fc882020823f..507364e95ea68 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -31,8 +31,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Do not depend on the repository or enterprise default token permissions. +permissions: + contents: read + jobs: preconditions: + permissions: + contents: read + pull-requests: read # List changed files and check whether the PR is ready. name: Preconditions runs-on: ubuntu-24.04 outputs: diff --git a/.github/workflows/ci-python-functions.yaml b/.github/workflows/ci-python-functions.yaml index d02ed5303ebde..eb67f184f6f6f 100644 --- a/.github/workflows/ci-python-functions.yaml +++ b/.github/workflows/ci-python-functions.yaml @@ -33,8 +33,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Do not depend on the repository or enterprise default token permissions. +permissions: + contents: read + jobs: preconditions: + permissions: + contents: read + pull-requests: read # List changed files and check whether the PR is ready. name: Preconditions runs-on: ubuntu-24.04 outputs: diff --git a/.github/workflows/ci-semantic-pull-request.yml b/.github/workflows/ci-semantic-pull-request.yml index bd552b4199601..a7dc2cd942532 100644 --- a/.github/workflows/ci-semantic-pull-request.yml +++ b/.github/workflows/ci-semantic-pull-request.yml @@ -29,6 +29,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} cancel-in-progress: true +permissions: + pull-requests: read + jobs: main: name: Check pull request title diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index ccaa8d1c87501..1b693dea9e351 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -36,7 +36,7 @@ env: jobs: analyze: # only run on push and schedule in apache/pulsar repo - if: ${{ github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch' }} + if: ${{ (github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch') && (github.event.repository.visibility == 'public' || vars.CI_ENABLE_CODEQL == 'true') }} name: Analyze runs-on: 'ubuntu-latest' timeout-minutes: 360 diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 6e18be2060cb2..28d5485655ae0 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -81,8 +81,16 @@ env: ARTIFACT_RETENTION_DAYS: 3 JDK_DISTRIBUTION: corretto +# Do not depend on the repository or enterprise default token permissions. +permissions: + contents: read + jobs: preconditions: + permissions: + contents: read + pull-requests: read # List changed files and check whether the PR is ready. + actions: write # Cancel scheduled runs in forks. name: Preconditions runs-on: ubuntu-24.04 outputs: diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 6e0c0e2335172..73305e5f85180 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -76,8 +76,16 @@ env: ARTIFACT_RETENTION_DAYS: 3 JDK_DISTRIBUTION: corretto +# Do not depend on the repository or enterprise default token permissions. +permissions: + contents: read + jobs: preconditions: + permissions: + contents: read + pull-requests: read # List changed files and check whether the PR is ready. + actions: write # Cancel scheduled runs in forks. name: Preconditions runs-on: ubuntu-24.04 outputs: @@ -638,6 +646,9 @@ jobs: action: wait pulsar-test-latest-version-image: + permissions: + contents: read + security-events: write # Upload scheduled container scan results. name: Build pulsar-test-latest-version Docker image runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -855,7 +866,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 needs: ['preconditions', 'unit-tests'] - if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} + # Private repositories need Code Security enabled before opting in. + if: ${{ (github.event.repository.visibility == 'public' || vars.CI_ENABLE_CODEQL == 'true') && (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} permissions: actions: read contents: read @@ -916,6 +928,8 @@ jobs: # protected_branches section for master branch required_status_checks. # It depends on all other jobs in this workflow. pulsar-ci-checks-completed: + permissions: + actions: write # Delete intermediate build artifacts. name: "Pulsar CI checks completed" # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8929d1990e534..ec3f0a2115266 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,6 +105,23 @@ Other test-related properties: `-PtestJavaVersion=17` (run tests on a different `-PtestRetryCount=N`, `-PtestFailFast=true|false`, `-PprotobufVersion=4.31.1` (protobuf v4 compatibility tests). +Test JVMs default to a `1300m` heap and up to four forks per task. Override these with +`-PtestMaxHeapSize=1500m` and `-PtestMaxParallelForks=2`; `--max-workers=2` also limits +concurrent workers across projects. Task-specific limits, such as the integration tests' single +fork and `1G` heap, take precedence. + +Use `-PtestForkEvery=50` to replace a test JVM after 50 detected test classes, limiting state +retained across classes. The default is `0` (no class limit). Gradle counts candidate classes +passed to test workers before TestNG group filtering, so classes excluded by groups still count +toward each batch. `--tests` can narrow candidates, but wildcard filters may still count classes +that execute no matching tests. Test methods, data-provider rows and factory instances do not +count separately. +Tasks using TestNG XML suites and async-profiler keep recycling disabled; task-specific +isolation, such as SASL's one class per worker, takes precedence. + +Test JVMs write heap dumps on heap exhaustion to `/tmp/java_pid.hprof`, collected by CI's +existing failure artifacts. Set `-PtestHeapDumpPath=` to use another directory. + Failed tests are retried once by default (`testRetryCount=1`; `0` when running inside the IDE). When running tests locally, prefer **`-PtestRetryCount=0`** to catch failures (including flakiness) early instead of having retries mask them. @@ -333,6 +350,24 @@ locally. (`integrationTest` also accepts `-PtestGroups` / `-PexcludedTestGroups` ### Running the full CI pipeline (Personal CI) +The shared `setup-gradle` action automatically selects a smaller memory profile on Linux runners +with up to 8 GiB of physical RAM: a `2g` Gradle heap, two workers across projects, up to two test +forks per task, and new test workers after 50 detected classes. Larger runners retain the usual +settings. The action's `memory-profile` input accepts `auto` (default), `low-memory` (force the +smaller profile), or `standard` (leave the memory settings unchanged). Command-line `-Dorg.gradle.jvmargs`, +`--max-workers` and `-Ptest*` options can override the profile settings. + +Develocity injection and build-scan publishing are disabled when the workflow repository is +private or its visibility is unavailable, even if an access key is configured. Public repositories +can also disable them with the action's `build-scan-publish: 'false'` input. CodeQL runs by default +for public repositories; private repositories with GitHub Code Security enabled can opt in by +setting the repository variable `CI_ENABLE_CODEQL=true`. + +The CI workflows declare the token permissions needed by their jobs, including reading PR changes. +This supports repositories whose organization or enterprise enforces read-only default workflow +permissions. Explicit permissions do not override restrictions on tokens for fork pull requests or +enterprise policies that prohibit particular actions. + The full test suite is large and slow to run locally. While iterating on a change, run only the narrowly-scoped tests relevant to the change (a single test class or package, see above) rather than a module's entire test task. To validate a larger change against the **full** CI pipeline, do not run diff --git a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts index 785ea7cf9df15..cbb8581320fb0 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts @@ -236,8 +236,9 @@ tasks.withType().configureEach { showExceptions = true showCauses = true } - maxHeapSize = "1300m" - maxParallelForks = 4 + maxHeapSize = providers.gradleProperty("testMaxHeapSize").getOrElse("1300m") + maxParallelForks = providers.gradleProperty("testMaxParallelForks").map { it.toInt() }.getOrElse(4) + forkEvery = providers.gradleProperty("testForkEvery").map { it.toLong() }.getOrElse(0L) val failFastValue = providers.gradleProperty("testFailFast").getOrElse("true").toBoolean() failFast = failFastValue val ideaActive = providers.systemProperty("idea.active").map { it.toBoolean() }.getOrElse(false) @@ -245,6 +246,8 @@ tasks.withType().configureEach { systemProperty("testRetryCount", providers.gradleProperty("testRetryCount").getOrElse(defaultTestRetryCount)) systemProperty("testFailFast", failFastValue.toString()) jvmArgs( + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:HeapDumpPath=${providers.gradleProperty("testHeapDumpPath").getOrElse("/tmp")}", "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.io=ALL-UNNAMED", @@ -368,6 +371,7 @@ if (asyncProfilerEnabled) { systemProperty("pulsar.test.enableManualTest", "true") // One test JVM at a time and no retries, so that a run produces a single comparable profile. maxParallelForks = 1 + forkEvery = 0 systemProperty("testRetryCount", "0") // A profiling run has to actually run the tests, even when the task is up-to-date. Don't // "fix" this by declaring inputs: the point is to re-run, not to track a missing input. The diff --git a/tests/pulsar-client-admin-shade-test/build.gradle.kts b/tests/pulsar-client-admin-shade-test/build.gradle.kts index 2619513ac2261..ab1a1af134dd6 100644 --- a/tests/pulsar-client-admin-shade-test/build.gradle.kts +++ b/tests/pulsar-client-admin-shade-test/build.gradle.kts @@ -48,6 +48,8 @@ dependencies { } tasks.named("test") { + // Each worker executes the full XML suite, so do not split it into class batches. + forkEvery = 0 useTestNG { suiteXmlFiles = listOf(file("src/test/resources/pulsar.xml")) } diff --git a/tests/pulsar-client-all-shade-test/build.gradle.kts b/tests/pulsar-client-all-shade-test/build.gradle.kts index db331e206851d..11e230b12c9bf 100644 --- a/tests/pulsar-client-all-shade-test/build.gradle.kts +++ b/tests/pulsar-client-all-shade-test/build.gradle.kts @@ -45,6 +45,8 @@ dependencies { } tasks.named("test") { + // Each worker executes the full XML suite, so do not split it into class batches. + forkEvery = 0 useTestNG { suiteXmlFiles = listOf(file("src/test/resources/pulsar.xml")) } diff --git a/tests/pulsar-client-native-image/build.gradle.kts b/tests/pulsar-client-native-image/build.gradle.kts index 4b484b05ef18c..37b0c887b136c 100644 --- a/tests/pulsar-client-native-image/build.gradle.kts +++ b/tests/pulsar-client-native-image/build.gradle.kts @@ -69,6 +69,8 @@ graalvmNative { } tasks.named("test") { + // Each worker executes the full XML suite, so do not split it into class batches. + forkEvery = 0 useTestNG { suiteXmlFiles = listOf(file("src/test/resources/native-image-tests.xml")) } diff --git a/tests/pulsar-client-shade-test/build.gradle.kts b/tests/pulsar-client-shade-test/build.gradle.kts index ad0b61bb06a1f..9f124f0e527dd 100644 --- a/tests/pulsar-client-shade-test/build.gradle.kts +++ b/tests/pulsar-client-shade-test/build.gradle.kts @@ -53,6 +53,8 @@ dependencies { } tasks.named("test") { + // Each worker executes the full XML suite, so do not split it into class batches. + forkEvery = 0 useTestNG { suiteXmlFiles = listOf(file("src/test/resources/pulsar.xml")) }