diff --git a/Jenkinsfile b/Jenkinsfile index 2c15e6a162d5..85ce9bb9d8d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,6 +99,21 @@ helpers.rootLinuxNode(env, { echo "Installing golangci-lint v2.13.2..." curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "${GOBIN}" v2.13.2 + # Install shellcheck for the Shellcheck stage + echo "Installing shellcheck v0.11.0..." + case "$(uname -m)" in + x86_64) sc_arch=x86_64; sc_sha=b7af85e41cc99489dcc21d66c6d5f3685138f06d34651e6d34b42ec6d54fe6f6 ;; + aarch64|arm64) sc_arch=aarch64; sc_sha=68a8133197a50beb8803f8d42f9908d1af1c5540d4bb05fdfca8c1fa47decefc ;; + *) echo "No shellcheck build for $(uname -m)" >&2; exit 1 ;; + esac + sc_tgz="$(mktemp)" + # This script has no `set -e`: never extract an unverified download. + curl -sSfL -o "${sc_tgz}" "https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.${sc_arch}.tar.gz" \\ + && echo "${sc_sha} ${sc_tgz}" | sha256sum -c - \\ + && tar -xzf "${sc_tgz}" -C "${GOBIN}" --strip-components=1 shellcheck-v0.11.0/shellcheck \\ + || { rm -f "${sc_tgz}"; echo "shellcheck install failed" >&2; exit 1; } + rm -f "${sc_tgz}" + # Set up Node source ~/.nvm/nvm.sh nvm install 24 && nvm use 24 && nvm alias default 24 @@ -174,11 +189,13 @@ helpers.rootLinuxNode(env, { def hasProtocolChanges = protocolChanges.size() != 0 def hasJSChanges = helpers.hasChanges('shared', env) def hasJenkinsfileChanges = helpers.getChanges(env.COMMIT_HASH, env.CHANGE_TARGET).findIndexOf{ name -> name =~ /Jenkinsfile/ } >= 0 + def hasShellcheckChanges = helpers.getChanges(env.COMMIT_HASH, env.CHANGE_TARGET).findIndexOf{ name -> name =~ /^packaging\/|^shared\/react-native\/(gobuild|packageAndBuild)\.sh$/ } >= 0 def hasKBFSChanges = false println "Has go changes: " + hasGoChanges println "Has protocol changes: " + hasProtocolChanges println "Has JS changes: " + hasJSChanges println "Has Jenkinsfile changes: " + hasJenkinsfileChanges + println "Has shellcheck changes: " + hasShellcheckChanges def dependencyFiles = [:] if (hasGoChanges && env.CHANGE_TARGET && !hasJenkinsfileChanges) { @@ -196,6 +213,17 @@ helpers.rootLinuxNode(env, { parallel ( failFast: true, test_linux: { + if (hasShellcheckChanges || hasJenkinsfileChanges) { + stage("Shellcheck") { + // Run from the repo root: the `# shellcheck source=` + // directives are relative to it. + sh ''' + shellcheck --version + git ls-files -z -- 'packaging/*.sh' shared/react-native/gobuild.sh shared/react-native/packageAndBuild.sh \\ + | xargs -0 shellcheck -S warning -x + ''' + } + } def packagesToTest = [:] if (hasProtocolChanges || hasJenkinsfileChanges) { // Clean the index first diff --git a/packaging/README.md b/packaging/README.md index eeb582c94005..bc288106516f 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -3,3 +3,40 @@ for the Keybase app. Most of this is only useful internally. If you're packaging for Linux, more information is available in the https://github.com/keybase/client/tree/master/packaging/linux directory. + +## Entry points + +The macOS, mobile and Windows builds are launched by the build bots in +[keybase/slackbot](https://github.com/keybase/slackbot) (`keybot/keybot.go`, +`keybot/winbot.go`). Check there before renaming a script or an environment +variable it reads. + +| Script | Run by | Inputs | +|--------|--------|--------| +| `build_darwin.sh` → `prerelease/pull_build.sh` → `prerelease/build_app.sh` | keybot, macOS build machine | `ARCH` (`amd64`/`arm64`, both if unset), `CLIENT_COMMIT`, `TEST`, `SMOKE_TEST`, `NOWAIT` (skip the CI gate), `NOPULL`, `NOS3`, `NONOTARIZE`, `NOBUILD`, `GOPATH` | +| `build_mobile.sh` | keybot, mobile build machine | runs the Android then the iOS script below | +| `android/build_and_publish.sh` | keybot (directly or via `build_mobile.sh`) | `CLIENT_COMMIT`, `CHECK_CI`, `AUTOMATED_BUILD`, `CACHE_NPM`, `CACHE_GO_LIB`, `ANDROID_*`/NDK paths | +| `ios/build_and_publish.sh` | keybot (directly or via `build_mobile.sh`) | `CLIENT_COMMIT`, `CHECK_CI`, `AUTOMATED_BUILD`, `CLEAN`, `CACHE_GO_LIB`, `GOPATH` | +| `windows/dorelease.cmd` | winbot, Windows build machine | see the script header | +| `linux/docker_build.sh MODE COMMIT` | Linux build host | `KEYBASE_RELEASE` / `KEYBASE_NIGHTLY` / `KEYBASE_TEST`, `KEYBASE_TEST_CODE_SIGNING_KEY`, `BUCKET_NAME`, `SECRETS_TEAM`, `SECRETS_REPO` | + +keybot also passes `KBFS_COMMIT` to `build_darwin.sh`; nothing reads it. + +Chat notifications go through `slack/send.sh`, which posts to Keybase chat when +`KEYBASE_CHAT_CONVID` is set (`KEYBASE_LOCATION` and `KEYBASE_HOME` pick the +binary and home dir). + +## Shared code + +`lib.sh` is sourced by the build scripts. It holds the pinned +KeybaseInstaller/KeybaseUpdater versions and checksums, `kb_fetch_verified`, +and the CI checks the release `wait-ci` gate waits for. + +## Linting + +The `Shellcheck` stage in the `Jenkinsfile` runs `shellcheck -S warning -x` over +every `.sh` file here plus `shared/react-native/gobuild.sh` and +`shared/react-native/packageAndBuild.sh`, whenever one of them (or the +`Jenkinsfile`) changes. The Jenkins setup step installs a pinned, checksummed +shellcheck (v0.11.0). Run it from the repo root so the `# shellcheck source=` +directives resolve. diff --git a/packaging/android/build_and_publish.sh b/packaging/android/build_and_publish.sh index 3a929f7fc017..4ed0bad4d825 100755 --- a/packaging/android/build_and_publish.sh +++ b/packaging/android/build_and_publish.sh @@ -3,10 +3,8 @@ set -eE -u -o pipefail # Fail on error, call ERR trap automated_build=${AUTOMATED_BUILD:-} -gopath=${GOPATH:-} dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) client_dir="$dir/../.." -kbfs_dir="$client_dir/go/kbfs" shared_dir="$client_dir/shared" android_dir="$shared_dir/android" cache_npm=${CACHE_NPM:-} diff --git a/packaging/binary_name.sh b/packaging/binary_name.sh index c25e78bb201a..60513d90067f 100755 --- a/packaging/binary_name.sh +++ b/packaging/binary_name.sh @@ -6,7 +6,7 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" mode="$("$here/build_mode.sh" "$@")" diff --git a/packaging/ios/build_and_publish.sh b/packaging/ios/build_and_publish.sh index 81e01378017a..b72c746511c0 100755 --- a/packaging/ios/build_and_publish.sh +++ b/packaging/ios/build_and_publish.sh @@ -8,7 +8,6 @@ client_dir="$gopath/src/github.com/keybase/client" shared_dir="$gopath/src/github.com/keybase/client/shared" ios_dir="$gopath/src/github.com/keybase/client/shared/ios" clean=${CLEAN:-} -cache_npm=${CACHE_NPM:-} cache_go_lib=${CACHE_GO_LIB:-} client_commit=${CLIENT_COMMIT:-} check_ci=${CHECK_CI:-1} diff --git a/packaging/lib.sh b/packaging/lib.sh index 2f6a7ada6f05..3f061ed06a6b 100644 --- a/packaging/lib.sh +++ b/packaging/lib.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # Shared helpers for the packaging scripts. Source it, don't run it. +# The variables here are read by the scripts that source this file. +# shellcheck disable=SC2034 # Native macOS apps bundled into Keybase.app. Bump the version and the sha256 # together: `shasum -a 256` of the tgz at the URL below. diff --git a/packaging/linux/arch/arch_common.sh b/packaging/linux/arch/arch_common.sh index 5bed6908d1c5..6363265c0ed2 100644 --- a/packaging/linux/arch/arch_common.sh +++ b/packaging/linux/arch/arch_common.sh @@ -13,7 +13,7 @@ setup_arch_build() { # build_binaries.sh and deb/package_binaries.sh must already have # been run on this build root. - here="$(dirname "$BASH_SOURCE")" + here="$(dirname "${BASH_SOURCE[0]}")" mkdir -p "$build_root/arch" diff --git a/packaging/linux/arch/build_test_package.sh b/packaging/linux/arch/build_test_package.sh index 27c3bb7cf831..94fc0cac2d0f 100755 --- a/packaging/linux/arch/build_test_package.sh +++ b/packaging/linux/arch/build_test_package.sh @@ -2,7 +2,8 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" +# shellcheck source=packaging/linux/arch/arch_common.sh source "$here/arch_common.sh" build_root="$1" diff --git a/packaging/linux/arch/keybase_git_version.sh b/packaging/linux/arch/keybase_git_version.sh index e051dccac366..e25bd9ce095b 100755 --- a/packaging/linux/arch/keybase_git_version.sh +++ b/packaging/linux/arch/keybase_git_version.sh @@ -9,7 +9,7 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" base_version="$("$here/../../version.sh" production)" commit_count="$(git -C "$here" rev-list --count HEAD)" diff --git a/packaging/linux/arch/update_aur_packages.sh b/packaging/linux/arch/update_aur_packages.sh index ef1cf602641a..29bbe5261701 100755 --- a/packaging/linux/arch/update_aur_packages.sh +++ b/packaging/linux/arch/update_aur_packages.sh @@ -4,7 +4,8 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" +# shellcheck source=packaging/linux/arch/arch_common.sh source "$here/arch_common.sh" build_root="$1" diff --git a/packaging/linux/deb/layout_repo.sh b/packaging/linux/deb/layout_repo.sh index e40769378566..ffe62c5908a4 100755 --- a/packaging/linux/deb/layout_repo.sh +++ b/packaging/linux/deb/layout_repo.sh @@ -41,6 +41,8 @@ Architectures: amd64 SignWith: $code_signing_fingerprint END +# One arch today; the loop keeps room for more. +# shellcheck disable=SC2043 for debian_arch in amd64 ; do echo Creating the Debian repo hierarchy. # Create/update the Debian repository hierarchy. The layout of this is diff --git a/packaging/linux/deb/package_binaries.sh b/packaging/linux/deb/package_binaries.sh index 59d68270ca84..3575b86cf039 100755 --- a/packaging/linux/deb/package_binaries.sh +++ b/packaging/linux/deb/package_binaries.sh @@ -44,7 +44,6 @@ else # We don't actually publish devel builds. This URL is a dream within a dream. repo_url="http://dist.keybase.io/linux/deb_devel/repo" fi -repo_ssl_url="$(echo "$repo_url" | sed 's|http|https|')" build_one_architecture() { echo "Making .deb package for $debian_arch." diff --git a/packaging/linux/docker/build.sh b/packaging/linux/docker/build.sh index 735166a1f186..fc4d28f425ed 100755 --- a/packaging/linux/docker/build.sh +++ b/packaging/linux/docker/build.sh @@ -16,7 +16,7 @@ gpg_tempfile="$client_dir/.docker/code_signing_key" gpg --export-secret-key --armor "$code_signing_fingerprint" > "$gpg_tempfile" # Don't store any secrets in the repo dir -trap "rm -r ""$client_dir/.docker"" || true" ERR +trap 'rm -r "$client_dir/.docker" || true' ERR # Load up all the config we need now, the rest will be resolved as needed config_file="$client_dir/packaging/linux/docker/config.json" diff --git a/packaging/linux/rpm/layout_repo.sh b/packaging/linux/rpm/layout_repo.sh index d52bf0d0f568..a37af8dc1aeb 100755 --- a/packaging/linux/rpm/layout_repo.sh +++ b/packaging/linux/rpm/layout_repo.sh @@ -54,6 +54,8 @@ fi # Copy the RPM files into our repo. This is flatter than the way Debian's # reprepro does things, and it's allowed to contain multiple copies of the same # package. +# One arch today; the loop keeps room for more. +# shellcheck disable=SC2043 for arch in x86_64 ; do rpmfile="$(ls "$build_root/rpm/$arch/RPMS/$arch"/*.rpm)" # keybase, kdstage, or kbdev rpmname="$(basename "$rpmfile")" diff --git a/packaging/linux/rpm/package_binaries.sh b/packaging/linux/rpm/package_binaries.sh index 4d6d9130803b..b78b6da50241 100755 --- a/packaging/linux/rpm/package_binaries.sh +++ b/packaging/linux/rpm/package_binaries.sh @@ -9,7 +9,7 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" build_root="${1:-}" if [ -z "$build_root" ] ; then @@ -40,7 +40,6 @@ else # dream. repo_url="http://dist.keybase.io/linux/rpm_devel/repo" fi -repo_ssl_url="$(echo "$repo_url" | sed 's|http|https|')" build_one_architecture() { echo "Making .rpm package for $rpm_arch." diff --git a/packaging/prerelease/build_app.sh b/packaging/prerelease/build_app.sh index dee705951b37..e575dc177aee 100755 --- a/packaging/prerelease/build_app.sh +++ b/packaging/prerelease/build_app.sh @@ -10,7 +10,6 @@ source "$dir/../lib.sh" gopath=${GOPATH:-} nobuild=${NOBUILD:-} # Don't build go binaries istest=${TEST:-} # If set to true, only build (for testing) -nopull=${NOPULL:-} # Don't git pull client_commit=${CLIENT_COMMIT:-} # Commit on client to build from bucket_name=${BUCKET_NAME:-"prerelease.keybase.io"} platform=${PLATFORM:-} # darwin,darwin-arm64,linux,windows (Only darwin/darwin-arm64 is supported in this script) @@ -18,7 +17,6 @@ nos3=${NOS3:-} # Don't sync to S3 nowait=${NOWAIT:-} # Don't wait for CI smoke_test=${SMOKE_TEST:-} # If set to 1, enable smoke testing skip_notarize=${NONOTARIZE:-} # Skip notarize -arch=${ARCH:-"amd64"} # architecture if [ "$gopath" = "" ]; then echo "No GOPATH" @@ -73,7 +71,7 @@ fi if [ ! "$nowait" = "1" ]; then echo "Checking client CI" - "$release_bin" wait-ci --repo="client" --commit=$(git -C "$client_dir" log -1 --pretty=format:%h) "${KB_CI_CONTEXTS[@]}" + "$release_bin" wait-ci --repo="client" --commit="$(git -C "$client_dir" log -1 --pretty=format:%h)" "${KB_CI_CONTEXTS[@]}" "$client_dir/packaging/slack/send.sh" "CI tests passed! Starting build for $platform." fi diff --git a/packaging/version.sh b/packaging/version.sh index 161aec5e7ca8..5b8856d5a845 100755 --- a/packaging/version.sh +++ b/packaging/version.sh @@ -7,10 +7,10 @@ set -e -u -o pipefail -here="$(dirname "$BASH_SOURCE")" +here="$(dirname "${BASH_SOURCE[0]}")" mode="$("$here/build_mode.sh" "$@")" -version_file="$(dirname "$BASH_SOURCE")/../go/libkb/version.go" +version_file="$here/../go/libkb/version.go" version="$(cat "$version_file" | grep 'Version =' | grep -oE '[0-9]+(.[0-9]+)+')" build="" diff --git a/shared/react-native/packageAndBuild.sh b/shared/react-native/packageAndBuild.sh index 7939b8fecc80..e70c5f3834da 100755 --- a/shared/react-native/packageAndBuild.sh +++ b/shared/react-native/packageAndBuild.sh @@ -31,14 +31,12 @@ requestbundles () return fi printf "\nRequesting bundles...\n" - case "ios" in - $arg1|$arg2) - curl -s -o /dev/null localhost:8081/index.bundle?platform=ios&dev=true&minify=false 2>&1;; - esac - case "android" in - $arg1|$arg2) - curl -s -o /dev/null localhost:8081/index.bundle?platform=android&dev=true&minify=false 2>&1;; - esac + if [ "$arg1" == "ios" ] || [ "$arg2" == "ios" ]; then + curl -s -o /dev/null "localhost:8081/index.bundle?platform=ios&dev=true&minify=false" 2>&1 || true + fi + if [ "$arg1" == "android" ] || [ "$arg2" == "android" ]; then + curl -s -o /dev/null "localhost:8081/index.bundle?platform=android&dev=true&minify=false" 2>&1 || true + fi } # start in background so we get the bundler dashboard