Skip to content

chore(ci): mirror commercial layer visibility #8422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dreamorosi wants to merge 2 commits into
base: develop
Choose a base branch
from
+181 −17
Open

chore(ci): mirror commercial layer visibility #8422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
feb92f7
chore(ci): mirror source layer visibility
dreamorosi Sep 3, 2026
604889b
chore(ci): enforce partition layer version parity
dreamorosi Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 62 additions & 11 deletions .github/workflows/layers_partition_balance.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ jobs:
for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do
NAME="${LAYER}-${ARCHITECTURE}"
METADATA="source/${VERSION}.json"
POLICY="source/${VERSION}.policy.json"
ZIP="source/${VERSION}.zip"

aws --region us-east-1 lambda get-layer-version-by-arn \
Expand All @@ -162,6 +163,17 @@ jobs:
LOCATION=$(jq -r '.Content.Location' "$METADATA")
curl --fail --location --retry 3 --retry-delay 2 --output "$ZIP" "$LOCATION"

if ! aws --region us-east-1 lambda get-layer-version-policy \
--layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${NAME}" \
--version-number "$VERSION" > "$POLICY" 2> policy-error.txt; then
if grep -q ResourceNotFoundException policy-error.txt; then
echo '{"Policy":null}' > "$POLICY"
else
cat policy-error.txt
exit 1
fi
fi

EXPECTED_SHA=$(jq -r '.Content.CodeSha256' "$METADATA")
ACTUAL_SHA=$(openssl dgst -sha256 -binary "$ZIP" | openssl enc -base64)
if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then
Expand Down Expand Up @@ -250,15 +262,26 @@ jobs:
mkdir -p target

has_public_permission() {
jq -e '
local result

if jq -e '
(.Policy | if type == "string" then fromjson else . end)
| any(.Statement[]?;
.Sid == "PublicLayer"
and .Effect == "Allow"
and ((.Action | if type == "array" then . else [.] end) | index("lambda:GetLayerVersion") != null)
.Effect == "Allow"
and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion"))
and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null)
and ((.Condition // {}) | length == 0)
)
' "$1" > /dev/null
' "$1" > /dev/null; then
return 0
else
result=$?
if (( result > 1 )); then
echo "Unable to evaluate layer policy $1 (jq exit ${result})"
exit 1
fi
return 1
fi
}

if ! aws --region "$REGION" lambda list-layer-versions \
Expand All @@ -276,6 +299,7 @@ jobs:

for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do
METADATA="source/${VERSION}.json"
SOURCE_POLICY="source/${VERSION}.policy.json"
ZIP="source/${VERSION}.zip"
TARGET_METADATA="target/${VERSION}.json"
TARGET_ARN="arn:${PARTITION}:lambda:${REGION}:${AWS_ACCOUNT}:layer:${NAME}:${VERSION}"
Expand Down Expand Up @@ -313,6 +337,12 @@ jobs:
exit 1
fi

SOURCE_IS_PUBLIC=false
if has_public_permission "$SOURCE_POLICY"; then
SOURCE_IS_PUBLIC=true
fi
echo "Commercial source ${NAME}:${VERSION} public: ${SOURCE_IS_PUBLIC}"

if [[ "$VERSION_EXISTS" == "true" ]]; then
echo "${NAME}:${VERSION} already exists in ${REGION} with the expected SHA"
else
Expand All @@ -323,13 +353,21 @@ jobs:
fi
fi

if [[ "$SOURCE_IS_PUBLIC" == "false" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "true" ]]; then
echo "${NAME}:${VERSION} in ${REGION} is public but its commercial source is private"
exit 1
fi

if [[ "$DRY_RUN" != "false" ]]; then
if [[ "$VERSION_EXISTS" == "false" ]]; then
echo "Would publish ${NAME}:${VERSION} to ${REGION}"
CURRENT_POSITION=$VERSION
elif [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
elif [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
echo "Would add public permission to ${NAME}:${VERSION} in ${REGION}"
fi
if [[ "$SOURCE_IS_PUBLIC" == "false" ]]; then
echo "Would keep ${NAME}:${VERSION} private in ${REGION}"
fi
continue
fi

Expand All @@ -352,7 +390,7 @@ jobs:
CURRENT_POSITION=$PUBLISHED_VERSION
fi

if [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
if [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
if ! aws --region "$REGION" lambda add-layer-version-permission \
--layer-name "$NAME" \
--statement-id PublicLayer \
Expand All @@ -375,11 +413,24 @@ jobs:
exit 1
fi

aws --region "$REGION" lambda get-layer-version-policy \
if ! aws --region "$REGION" lambda get-layer-version-policy \
--layer-name "$NAME" \
--version-number "$VERSION" > scratch/policy.json
if ! has_public_permission scratch/policy.json; then
echo "${NAME}:${VERSION} in ${REGION} is missing the expected public permission"
--version-number "$VERSION" > scratch/policy.json 2> scratch/policy-error.txt; then
if grep -q ResourceNotFoundException scratch/policy-error.txt; then
echo '{"Policy":null}' > scratch/policy.json
else
cat scratch/policy-error.txt
exit 1
fi
fi

TARGET_IS_PUBLIC=false
if has_public_permission scratch/policy.json; then
TARGET_IS_PUBLIC=true
fi

if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then
echo "${NAME}:${VERSION} in ${REGION} does not match its commercial source visibility"
exit 1
fi
done
Expand Down
125 changes: 119 additions & 6 deletions .github/workflows/layers_partitions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,21 @@ jobs:
run: |
aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} --query 'Content.Location' | xargs curl -L -o ${{ matrix.layer }}-${{ matrix.arch }}.zip
aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} > ${{ matrix.layer }}-${{ matrix.arch }}.json
- name: Grab Policy
env:
LAYER_NAME: ${{ matrix.layer }}-${{ matrix.arch }}
VERSION: ${{ inputs.version }}
run: |
if ! aws --region us-east-1 lambda get-layer-version-policy \
--layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${LAYER_NAME}" \
--version-number "$VERSION" > "${LAYER_NAME}.policy.json" 2> policy-error.txt; then
if grep -q ResourceNotFoundException policy-error.txt; then
echo '{"Policy":null}' > "${LAYER_NAME}.policy.json"
else
cat policy-error.txt
exit 1
fi
fi
- name: Store Zip
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
Expand All @@ -112,6 +127,13 @@ jobs:
path: ${{ matrix.layer }}-${{ matrix.arch }}.json
retention-days: 1
if-no-files-found: error
- name: Store Policy
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json
path: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json
retention-days: 1
if-no-files-found: error

copy:
name: Copy
Expand Down Expand Up @@ -146,10 +168,37 @@ jobs:
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ matrix.layer }}-${{ matrix.arch }}.json
- name: Download Policy
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json
- name: Verify Layer Signature
run: |
SHA=$(jq -r '.Content.CodeSha256' '${{ matrix.layer }}-${{ matrix.arch }}.json')
test "$(openssl dgst -sha256 -binary ${{ matrix.layer }}-${{ matrix.arch }}.zip | openssl enc -base64)" == "$SHA" && echo "SHA OK: ${SHA}" || exit 1
- id: source_policy
name: Source Layer Visibility
run: |
if jq -e '
(.Policy | if type == "string" then fromjson else . end)
| any(.Statement[]?;
.Effect == "Allow"
and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion"))
and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null)
and ((.Condition // {}) | length == 0)
)
' '${{ matrix.layer }}-${{ matrix.arch }}.policy.json' > /dev/null; then
echo 'public=true' >> "$GITHUB_OUTPUT"
echo 'Commercial source layer is public'
else
RESULT=$?
if (( RESULT > 1 )); then
echo "Unable to evaluate source layer policy (jq exit ${RESULT})"
exit 1
fi
echo 'public=false' >> "$GITHUB_OUTPUT"
echo 'Commercial source layer is private'
fi
- id: transform
run: |
echo 'CONVERTED_REGION=${{ matrix.region }}' | tr 'a-z\-' 'A-Z_' >> "$GITHUB_OUTPUT"
Expand All @@ -160,8 +209,29 @@ jobs:
aws-region: ${{ matrix.region}}
mask-aws-account-id: true
audience: ${{ needs.setup.outputs.aud }}
- name: Validate target account
env:
AWS_ACCOUNT: ${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}
PARTITION: ${{ needs.setup.outputs.partition }}
REGION: ${{ matrix.region }}
run: |
if [[ ! "$AWS_ACCOUNT" =~ ^[0-9]{12}$ ]]; then
echo "AWS account secret for ${REGION} is missing or invalid"
exit 1
fi

CALLER_ACCOUNT=$(aws --region "$REGION" sts get-caller-identity --query Account --output text)
CALLER_ARN=$(aws --region "$REGION" sts get-caller-identity --query Arn --output text)

if [[ "$CALLER_ACCOUNT" != "$AWS_ACCOUNT" ]] || [[ "$CALLER_ARN" != "arn:${PARTITION}:"* ]]; then
echo "Assumed role does not match the expected account and partition for ${REGION}"
exit 1
fi
- name: Create Layer
id: create-layer
env:
SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }}
VERSION: ${{ inputs.version }}
run: |
jq '{"LayerName": "${{ matrix.layer }}-${{ matrix.arch }}", "Description": .Description, "CompatibleRuntimes": .CompatibleRuntimes, "CompatibleArchitectures": .CompatibleArchitectures, "LicenseInfo": .LicenseInfo} | with_entries(select(.value != null))' '${{ matrix.layer }}-${{ matrix.arch }}.json' > input.json

Expand All @@ -171,17 +241,25 @@ jobs:
--query 'Version' \
--output text)

if (( LAYER_VERSION != VERSION )); then
echo "Expected to publish as version ${VERSION}, received ${LAYER_VERSION}"
exit 1
fi

echo "LAYER_VERSION=$LAYER_VERSION" >> "$GITHUB_OUTPUT"

aws --region ${{ matrix.region}} lambda add-layer-version-permission \
--layer-name ${{ matrix.layer }}-${{ matrix.arch }} \
--statement-id 'PublicLayer' \
--action lambda:GetLayerVersion \
--principal '*' \
--version-number "$LAYER_VERSION"
if [[ "$SOURCE_IS_PUBLIC" == "true" ]]; then
aws --region ${{ matrix.region}} lambda add-layer-version-permission \
--layer-name ${{ matrix.layer }}-${{ matrix.arch }} \
--statement-id 'PublicLayer' \
--action lambda:GetLayerVersion \
--principal '*' \
--version-number "$LAYER_VERSION"
fi
- name: Verify Layer
env:
LAYER_VERSION: ${{ steps.create-layer.outputs.LAYER_VERSION }}
SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }}
run: |
export layer_output='${{ matrix.layer }}-${{ matrix.arch }}-${{matrix.region}}.json'
aws --region ${{ matrix.region}} lambda get-layer-version-by-arn --arn 'arn:${{ needs.setup.outputs.partition }}:lambda:${{ matrix.region}}:${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ env.LAYER_VERSION }}' > $layer_output
Expand All @@ -190,6 +268,41 @@ jobs:
test "$REMOTE_SHA" == "$LOCAL_SHA" && echo "SHA OK: ${LOCAL_SHA}" || exit 1
jq -s -r '["Layer Arn", "Runtimes", "Version", "Description", "SHA256"], ([.[0], .[1]] | .[] | [.LayerArn, (.CompatibleRuntimes | join("/")), .Version, .Description, .Content.CodeSha256]) |@tsv' '${{ matrix.layer }}-${{ matrix.arch }}.json' $layer_output | column -t -s $'\t'

if ! aws --region ${{ matrix.region}} lambda get-layer-version-policy \
--layer-name '${{ matrix.layer }}-${{ matrix.arch }}' \
--version-number "$LAYER_VERSION" > target-policy.json 2> policy-error.txt; then
if grep -q ResourceNotFoundException policy-error.txt; then
echo '{"Policy":null}' > target-policy.json
else
cat policy-error.txt
exit 1
fi
fi

TARGET_IS_PUBLIC=false
if jq -e '
(.Policy | if type == "string" then fromjson else . end)
| any(.Statement[]?;
.Effect == "Allow"
and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion"))
and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null)
and ((.Condition // {}) | length == 0)
)
' target-policy.json > /dev/null; then
TARGET_IS_PUBLIC=true
else
RESULT=$?
if (( RESULT > 1 )); then
echo "Unable to evaluate target layer policy (jq exit ${RESULT})"
exit 1
fi
fi

if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then
echo "Published layer visibility does not match its commercial source"
exit 1
fi

- name: Store Metadata - ${{ matrix.region }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
Expand Down