diff --git a/.github/workflows/layers_partition_balance.yml b/.github/workflows/layers_partition_balance.yml index abd23c42f08..76cc6a581fd 100644 --- a/.github/workflows/layers_partition_balance.yml +++ b/.github/workflows/layers_partition_balance.yml @@ -154,6 +154,7 @@ jobs: for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do NAME="${LAYER}-${ARCHITECTURE}" METADATA="source/${VERSION}.json" + POLICY="source/${VERSION}.policy.json" ZIP="source/${VERSION}.zip" aws --region us-east-1 lambda get-layer-version-by-arn \ @@ -162,6 +163,17 @@ jobs: LOCATION=$(jq -r '.Content.Location' "$METADATA") curl --fail --location --retry 3 --retry-delay 2 --output "$ZIP" "$LOCATION" + if ! aws --region us-east-1 lambda get-layer-version-policy \ + --layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${NAME}" \ + --version-number "$VERSION" > "$POLICY" 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > "$POLICY" + else + cat policy-error.txt + exit 1 + fi + fi + EXPECTED_SHA=$(jq -r '.Content.CodeSha256' "$METADATA") ACTUAL_SHA=$(openssl dgst -sha256 -binary "$ZIP" | openssl enc -base64) if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then @@ -250,15 +262,26 @@ jobs: mkdir -p target has_public_permission() { - jq -e ' + local result + + if jq -e ' (.Policy | if type == "string" then fromjson else . end) | any(.Statement[]?; - .Sid == "PublicLayer" - and .Effect == "Allow" - and ((.Action | if type == "array" then . else [.] end) | index("lambda:GetLayerVersion") != null) + .Effect == "Allow" + and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion")) and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null) + and ((.Condition // {}) | length == 0) ) - ' "$1" > /dev/null + ' "$1" > /dev/null; then + return 0 + else + result=$? + if (( result > 1 )); then + echo "Unable to evaluate layer policy $1 (jq exit ${result})" + exit 1 + fi + return 1 + fi } if ! aws --region "$REGION" lambda list-layer-versions \ @@ -276,6 +299,7 @@ jobs: for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do METADATA="source/${VERSION}.json" + SOURCE_POLICY="source/${VERSION}.policy.json" ZIP="source/${VERSION}.zip" TARGET_METADATA="target/${VERSION}.json" TARGET_ARN="arn:${PARTITION}:lambda:${REGION}:${AWS_ACCOUNT}:layer:${NAME}:${VERSION}" @@ -313,6 +337,12 @@ jobs: exit 1 fi + SOURCE_IS_PUBLIC=false + if has_public_permission "$SOURCE_POLICY"; then + SOURCE_IS_PUBLIC=true + fi + echo "Commercial source ${NAME}:${VERSION} public: ${SOURCE_IS_PUBLIC}" + if [[ "$VERSION_EXISTS" == "true" ]]; then echo "${NAME}:${VERSION} already exists in ${REGION} with the expected SHA" else @@ -323,13 +353,21 @@ jobs: fi fi + if [[ "$SOURCE_IS_PUBLIC" == "false" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "true" ]]; then + echo "${NAME}:${VERSION} in ${REGION} is public but its commercial source is private" + exit 1 + fi + if [[ "$DRY_RUN" != "false" ]]; then if [[ "$VERSION_EXISTS" == "false" ]]; then echo "Would publish ${NAME}:${VERSION} to ${REGION}" CURRENT_POSITION=$VERSION - elif [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then + elif [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then echo "Would add public permission to ${NAME}:${VERSION} in ${REGION}" fi + if [[ "$SOURCE_IS_PUBLIC" == "false" ]]; then + echo "Would keep ${NAME}:${VERSION} private in ${REGION}" + fi continue fi @@ -352,7 +390,7 @@ jobs: CURRENT_POSITION=$PUBLISHED_VERSION fi - if [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then + if [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then if ! aws --region "$REGION" lambda add-layer-version-permission \ --layer-name "$NAME" \ --statement-id PublicLayer \ @@ -375,11 +413,24 @@ jobs: exit 1 fi - aws --region "$REGION" lambda get-layer-version-policy \ + if ! aws --region "$REGION" lambda get-layer-version-policy \ --layer-name "$NAME" \ - --version-number "$VERSION" > scratch/policy.json - if ! has_public_permission scratch/policy.json; then - echo "${NAME}:${VERSION} in ${REGION} is missing the expected public permission" + --version-number "$VERSION" > scratch/policy.json 2> scratch/policy-error.txt; then + if grep -q ResourceNotFoundException scratch/policy-error.txt; then + echo '{"Policy":null}' > scratch/policy.json + else + cat scratch/policy-error.txt + exit 1 + fi + fi + + TARGET_IS_PUBLIC=false + if has_public_permission scratch/policy.json; then + TARGET_IS_PUBLIC=true + fi + + if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then + echo "${NAME}:${VERSION} in ${REGION} does not match its commercial source visibility" exit 1 fi done diff --git a/.github/workflows/layers_partitions.yml b/.github/workflows/layers_partitions.yml index 65b80f30748..55492733625 100644 --- a/.github/workflows/layers_partitions.yml +++ b/.github/workflows/layers_partitions.yml @@ -98,6 +98,21 @@ jobs: run: | aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} --query 'Content.Location' | xargs curl -L -o ${{ matrix.layer }}-${{ matrix.arch }}.zip aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} > ${{ matrix.layer }}-${{ matrix.arch }}.json + - name: Grab Policy + env: + LAYER_NAME: ${{ matrix.layer }}-${{ matrix.arch }} + VERSION: ${{ inputs.version }} + run: | + if ! aws --region us-east-1 lambda get-layer-version-policy \ + --layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${LAYER_NAME}" \ + --version-number "$VERSION" > "${LAYER_NAME}.policy.json" 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > "${LAYER_NAME}.policy.json" + else + cat policy-error.txt + exit 1 + fi + fi - name: Store Zip uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -112,6 +127,13 @@ jobs: path: ${{ matrix.layer }}-${{ matrix.arch }}.json retention-days: 1 if-no-files-found: error + - name: Store Policy + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json + path: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json + retention-days: 1 + if-no-files-found: error copy: name: Copy @@ -146,10 +168,37 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ matrix.layer }}-${{ matrix.arch }}.json + - name: Download Policy + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json - name: Verify Layer Signature run: | SHA=$(jq -r '.Content.CodeSha256' '${{ matrix.layer }}-${{ matrix.arch }}.json') test "$(openssl dgst -sha256 -binary ${{ matrix.layer }}-${{ matrix.arch }}.zip | openssl enc -base64)" == "$SHA" && echo "SHA OK: ${SHA}" || exit 1 + - id: source_policy + name: Source Layer Visibility + run: | + if jq -e ' + (.Policy | if type == "string" then fromjson else . end) + | any(.Statement[]?; + .Effect == "Allow" + and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion")) + and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null) + and ((.Condition // {}) | length == 0) + ) + ' '${{ matrix.layer }}-${{ matrix.arch }}.policy.json' > /dev/null; then + echo 'public=true' >> "$GITHUB_OUTPUT" + echo 'Commercial source layer is public' + else + RESULT=$? + if (( RESULT > 1 )); then + echo "Unable to evaluate source layer policy (jq exit ${RESULT})" + exit 1 + fi + echo 'public=false' >> "$GITHUB_OUTPUT" + echo 'Commercial source layer is private' + fi - id: transform run: | echo 'CONVERTED_REGION=${{ matrix.region }}' | tr 'a-z\-' 'A-Z_' >> "$GITHUB_OUTPUT" @@ -160,8 +209,29 @@ jobs: aws-region: ${{ matrix.region}} mask-aws-account-id: true audience: ${{ needs.setup.outputs.aud }} + - name: Validate target account + env: + AWS_ACCOUNT: ${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }} + PARTITION: ${{ needs.setup.outputs.partition }} + REGION: ${{ matrix.region }} + run: | + if [[ ! "$AWS_ACCOUNT" =~ ^[0-9]{12}$ ]]; then + echo "AWS account secret for ${REGION} is missing or invalid" + exit 1 + fi + + CALLER_ACCOUNT=$(aws --region "$REGION" sts get-caller-identity --query Account --output text) + CALLER_ARN=$(aws --region "$REGION" sts get-caller-identity --query Arn --output text) + + if [[ "$CALLER_ACCOUNT" != "$AWS_ACCOUNT" ]] || [[ "$CALLER_ARN" != "arn:${PARTITION}:"* ]]; then + echo "Assumed role does not match the expected account and partition for ${REGION}" + exit 1 + fi - name: Create Layer id: create-layer + env: + SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }} + VERSION: ${{ inputs.version }} run: | jq '{"LayerName": "${{ matrix.layer }}-${{ matrix.arch }}", "Description": .Description, "CompatibleRuntimes": .CompatibleRuntimes, "CompatibleArchitectures": .CompatibleArchitectures, "LicenseInfo": .LicenseInfo} | with_entries(select(.value != null))' '${{ matrix.layer }}-${{ matrix.arch }}.json' > input.json @@ -171,17 +241,25 @@ jobs: --query 'Version' \ --output text) + if (( LAYER_VERSION != VERSION )); then + echo "Expected to publish as version ${VERSION}, received ${LAYER_VERSION}" + exit 1 + fi + echo "LAYER_VERSION=$LAYER_VERSION" >> "$GITHUB_OUTPUT" - aws --region ${{ matrix.region}} lambda add-layer-version-permission \ - --layer-name ${{ matrix.layer }}-${{ matrix.arch }} \ - --statement-id 'PublicLayer' \ - --action lambda:GetLayerVersion \ - --principal '*' \ - --version-number "$LAYER_VERSION" + if [[ "$SOURCE_IS_PUBLIC" == "true" ]]; then + aws --region ${{ matrix.region}} lambda add-layer-version-permission \ + --layer-name ${{ matrix.layer }}-${{ matrix.arch }} \ + --statement-id 'PublicLayer' \ + --action lambda:GetLayerVersion \ + --principal '*' \ + --version-number "$LAYER_VERSION" + fi - name: Verify Layer env: LAYER_VERSION: ${{ steps.create-layer.outputs.LAYER_VERSION }} + SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }} run: | export layer_output='${{ matrix.layer }}-${{ matrix.arch }}-${{matrix.region}}.json' aws --region ${{ matrix.region}} lambda get-layer-version-by-arn --arn 'arn:${{ needs.setup.outputs.partition }}:lambda:${{ matrix.region}}:${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ env.LAYER_VERSION }}' > $layer_output @@ -190,6 +268,41 @@ jobs: test "$REMOTE_SHA" == "$LOCAL_SHA" && echo "SHA OK: ${LOCAL_SHA}" || exit 1 jq -s -r '["Layer Arn", "Runtimes", "Version", "Description", "SHA256"], ([.[0], .[1]] | .[] | [.LayerArn, (.CompatibleRuntimes | join("/")), .Version, .Description, .Content.CodeSha256]) |@tsv' '${{ matrix.layer }}-${{ matrix.arch }}.json' $layer_output | column -t -s $'\t' + if ! aws --region ${{ matrix.region}} lambda get-layer-version-policy \ + --layer-name '${{ matrix.layer }}-${{ matrix.arch }}' \ + --version-number "$LAYER_VERSION" > target-policy.json 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > target-policy.json + else + cat policy-error.txt + exit 1 + fi + fi + + TARGET_IS_PUBLIC=false + if jq -e ' + (.Policy | if type == "string" then fromjson else . end) + | any(.Statement[]?; + .Effect == "Allow" + and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion")) + and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null) + and ((.Condition // {}) | length == 0) + ) + ' target-policy.json > /dev/null; then + TARGET_IS_PUBLIC=true + else + RESULT=$? + if (( RESULT > 1 )); then + echo "Unable to evaluate target layer policy (jq exit ${RESULT})" + exit 1 + fi + fi + + if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then + echo "Published layer visibility does not match its commercial source" + exit 1 + fi + - name: Store Metadata - ${{ matrix.region }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: