Add upstream release gates and manifest metadata

.github/workflows/release-libpdfium.yml

@@ -15,13 +15,74 @@ on:
         type: boolean
         required: false
         default: false
+      upstream_gate_override_reason:
+        description: >-
+          Publish while missing upstream PDFium commits older than 2 days: why.
+          The release notes list them; likely security fixes must be cherry-picked
+          (docs/embedpdf/upstream-sync.md)
+        required: false
+        default: ''
 
 permissions:
   contents: read
 
 jobs:
+  # The release gates (docs/embedpdf/upstream-sync.md, "Release gates").
+  upstream-gate:
+    name: Upstream freshness and fork tags
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.ref_name }}
+          fetch-depth: 0
+
+      - name: Fetch upstream PDFium
+        shell: bash
+        run: |
+          for attempt in 1 2 3 4 5; do
+            if git fetch --no-tags https://pdfium.googlesource.com/pdfium main:refs/remotes/upstream/main; then
+              exit 0
+            fi
+            sleep $((attempt * 30))
+          done
+          exit 1
+
+      # Enforced only when publishing: sync branches are behind upstream on
+      # purpose until they merge.
+      - name: Check upstream freshness
+        shell: bash
+        env:
+          ENFORCE: ${{ inputs.release == true }}
+          OVERRIDE_REASON: ${{ inputs.upstream_gate_override_reason }}
+        run: |
+          mkdir -p out/upstream-gate
+          args=(--notes out/upstream-gate/notes.md --json out/upstream-gate/upstream.json)
+          if [[ "$ENFORCE" == true ]]; then
+            args+=(--enforce)
+          fi
+          python3 scripts/embedpdf-runtime/check-upstream-freshness.py "${args[@]}" \
+            --override-reason "$OVERRIDE_REASON"
+
+      - name: Check fork tags
+        shell: bash
+        run: python3 scripts/embedpdf-runtime/check-fork-tags.py --strict
+
+      - uses: actions/upload-artifact@v6
+        with:
+          name: upstream-gate
+          path: out/upstream-gate/
+
+  tsan-soak:
+    name: ThreadSanitizer soak
+    needs: upstream-gate
+    uses: ./.github/workflows/pdfium-tsan-thread-soak.yml
+    with:
+      ref: ${{ inputs.ref || github.ref_name }}
+
   source-tests:
     name: Source tests (Linux x64)
+    needs: upstream-gate
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v6
@@ -51,6 +112,12 @@ jobs:
         shell: bash
         run: scripts/embedpdf-runtime/test-target.sh linux-x64
 
+      - name: Run embedder tests in layer mode
+        shell: bash
+        run: |
+          python3 testing/tools/run_layer_mode.py out/embedpdf-runtime-tests-linux-x64 \
+            --out out/embedpdf-runtime-test-results/linux-x64/layer-mode
+
       - name: Upload PDFium test results
         if: always()
         uses: actions/upload-artifact@v6
@@ -168,6 +235,11 @@ jobs:
       - name: Build target
         shell: bash
         run: scripts/embedpdf-runtime/build-target.sh "${{ matrix.target }}"
+      # The native server runs this target on several threads in one process.
+      - name: Check writable globals
+        if: matrix.target == 'linux-x64'
+        shell: bash
+        run: python3 scripts/embedpdf-runtime/check-globals.py out/embedpdf-runtime/linux-x64/libembedpdf.so
       - name: Package target
         shell: bash
         run: scripts/embedpdf-runtime/package-target.sh "${{ matrix.target }}"
@@ -179,7 +251,7 @@ jobs:
   release:
     name: Publish release
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [upstream-gate, tsan-soak, source-tests, build]
     if: github.event_name == 'workflow_dispatch' && inputs.release == true
     permissions:
       contents: write
@@ -195,16 +267,24 @@ jobs:
         run: |
           mkdir -p out/embedpdf-runtime-artifacts
           find artifacts -name '*.tar.gz' -exec cp {} out/embedpdf-runtime-artifacts/ \;
+      # The built ref's commit, which can differ from the commit the workflow
+      # was dispatched from.
+      - name: Name the release
+        id: name
+        shell: bash
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
       - name: Write manifest
         shell: bash
         run: |
-          tag="runtime-$(git rev-parse HEAD)"
+          tag="runtime-${{ steps.name.outputs.sha }}"
           export PDF_RUNTIME_ARTIFACT_DIR="$PWD/out/embedpdf-runtime-artifacts"
           export PDF_RUNTIME_RELEASE_BASE_URL="https://github.com/${{ github.repository }}/releases/download/$tag"
+          export PDF_RUNTIME_UPSTREAM_FILE="$PWD/artifacts/upstream-gate/upstream.json"
           scripts/embedpdf-runtime/write-manifest.sh "$PDF_RUNTIME_ARTIFACT_DIR/pdf-runtime-build.generated.json"
       - uses: ncipollo/release-action@v1
         with:
-          tag: runtime-${{ github.sha }}
-          name: EmbedPDF PDF Runtime ${{ github.sha }}
+          tag: runtime-${{ steps.name.outputs.sha }}
+          name: EmbedPDF PDF Runtime ${{ steps.name.outputs.sha }}
+          bodyFile: artifacts/upstream-gate/notes.md
           artifacts: out/embedpdf-runtime-artifacts/*
           allowUpdates: true

.github/workflows/upstream-mirror.yml

@@ -35,23 +35,4 @@ jobs:
 
       - name: Report what embedpdf/main hasn't merged
         shell: bash
-        run: |
-          set -euo pipefail
-          base=$(git merge-base HEAD upstream/main)
-          count=$(git rev-list --count "$base..upstream/main")
-          oldest=$(git log --reverse --format=%ci "$base..upstream/main" | head -1)
-          {
-            echo "## Upstream PDFium not in embedpdf/main"
-            echo
-            echo "- Base: \`$(git log -1 --format='%h %ad %s' --date=short "$base")\`"
-            echo "- Unmerged upstream commits: **$count**${oldest:+, the oldest from $oldest}"
-            echo
-            echo "### Likely security fixes"
-            echo
-          } >> "$GITHUB_STEP_SUMMARY"
-          git log --format='%h%x09%ad%x09%s' --date=short "$base..upstream/main" |
-            grep -iE 'overflow|crash|out[- ]of[- ]bounds|use[- ]after[- ]free|heap|stack[- ]buffer|null deref|fuzz|security|sanitiz' |
-            while IFS=$'\t' read -r hash date subject; do
-              echo "- \`$hash\` $date $subject" >> "$GITHUB_STEP_SUMMARY"
-              echo "::warning title=Likely upstream security fix::$hash $subject"
-            done || true
+        run: python3 scripts/embedpdf-runtime/check-upstream-freshness.py

docs/embedpdf/upstream-sync.md

@@ -75,6 +75,26 @@ All commands run from the fork's root.
      `BUILD.gn.orig`, so only the merge is committed.
 7. **Open a PR into `embedpdf/main`** with the report.
 
+## Release gates
+
+`release-libpdfium.yml` runs these before anything is published, on every run:
+
+- **Upstream freshness** (`check-upstream-freshness.py`): lists the upstream
+  commits the ref misses, likely security fixes marked. With `release` on it
+  fails when the oldest one landed more than 2 days ago. The release notes carry
+  the list, and the manifest the upstream commit the release is based on.
+- **Fork tags** (`check-fork-tags.py --strict`).
+- **Layer mode** on the Linux test build (`run_layer_mode.py`).
+- **Writable globals** on the linux-x64 library (`check-globals.py`).
+- **ThreadSanitizer soak** (`pdfium-tsan-thread-soak.yml`).
+
+**The override.** When a sync can't land before a release, set
+`upstream_gate_override_reason`. The release then publishes with a warning, and
+its notes give the reason and every upstream commit it misses. Every likely
+security fix it holds back must be cherry-picked into the release first, with
+`git cherry-pick -x` (the gate checks for the `cherry picked from commit` line).
+The next release tries the full sync again.
+
 ## Never silently
 
 - Take upstream's side of a tagged block without a line in the report.

scripts/embedpdf-runtime/check-globals.py

@@ -51,8 +51,9 @@
 THREAD_LOCAL_SECTIONS = ('.tdata', '.tbss', '__DATA,__thread_vars')
 
 # `<address> <flags> O <section> [<size>] [.hidden] <name>`; ELF has the size.
-SYMBOL = re.compile(r'^[0-9a-f]+\s.*?\sO\s+(\S+)\s+(?:[0-9a-f]{8,}\s+)?'
-                    r'(?:\.hidden\s+)?(.+)$')
+# ELF thread-local symbols (STT_TLS) have no `O` flag: `<address> l .tbss ...`.
+SYMBOL = re.compile(r'^[0-9a-f]+\s.*?\s(?:O\s+(\S+)|(\.tbss|\.tdata))\s+'
+                    r'(?:[0-9a-f]{8,}\s+)?(?:\.hidden\s+)?(.+)$')
 
 
 def normalize(name, macho):
@@ -76,7 +77,10 @@ def read_symbols(library):
     m = SYMBOL.match(line)
     if not m:
       continue
-    section, name = m.group(1), normalize(m.group(2).strip(), macho)
+    section = m.group(1) or m.group(2)
+    name = normalize(m.group(3).strip(), macho)
+    if name == section:  # the section's own symbol
+      continue
     if section in SHARED_SECTIONS:
       shared.add(name)
     elif section in THREAD_LOCAL_SECTIONS:

scripts/embedpdf-runtime/check-upstream-freshness.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# Copyright 2026 CloudPDF LTD
+# SPDX-License-Identifier: Apache-2.0
+
+"""Check that HEAD misses no upstream PDFium commit older than 2 days.
+
+The release gate of docs/embedpdf/upstream-sync.md: a release may not miss an
+upstream commit that was more than 2 days old when the release started. It
+lists the upstream commits HEAD doesn't contain (`git log <upstream> ^HEAD`),
+marks the ones that look like security fixes, and finds the upstream commit
+HEAD is based on (the merge base), which the release manifest records.
+
+It fails, with --enforce, when the oldest missing commit is older than
+--max-age-hours. With --override-reason it warns instead, but a missing commit
+that looks like a security fix must still be in HEAD as a cherry-pick: a commit
+since the base whose message says `(cherry picked from commit <sha>)`
+(`git cherry-pick -x`). Without --enforce it only reports, as the mirror job and
+release=false runs do.
+
+Commit times are committer dates: when the commit landed upstream.
+
+Usage:
+  check-upstream-freshness.py [--upstream REF] [--max-age-hours N] [--enforce]
+                              [--override-reason TEXT] [--notes FILE]
+                              [--json FILE]
+
+  --upstream         upstream main, fetched. Default: upstream/main.
+  --notes            write the report as Markdown (release notes, run summary).
+  --json             write {sha, date, subject} of the upstream base, plus the
+                     missing count and the override, for the release manifest.
+
+In GitHub Actions the report also goes to the run summary, and each likely
+security fix becomes a warning.
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+
+SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+
+# Subjects that look like security fixes; the mirror job runs this script too.
+# Not "fuzz" (fuzzer build changes, fuzzy image matching: fixes a fuzzer found
+# say crash or overflow) nor a bare "heap" (heap allocators).
+SECURITY = re.compile(
+    r'overflow|crash|out[- ]of[- ]bounds|\boob\b|use[- ]after[- ]free|\buaf\b|'
+    r'(?:heap|stack)[- ]buffer|null(?:ptr)? deref|security|sanitiz|\bcve\b',
+    re.IGNORECASE)
+CHERRY_PICK = re.compile(r'\(cherry picked from commit ([0-9a-f]{40})\)')
+
+
+def git(*args):
+  return subprocess.run(['git', '-C', SOURCE_DIR, *args],
+                        check=True,
+                        capture_output=True,
+                        text=True,
+                        errors='replace').stdout
+
+
+def commits(*rev_args):
+  """Returns [(sha, unix time, ISO date, subject)], newest first."""
+  out = git('log', '--format=%H%x09%ct%x09%cI%x09%s', *rev_args)
+  result = []
+  for line in out.splitlines():
+    sha, stamp, date, subject = line.split('\t', 3)
+    result.append((sha, int(stamp), date, subject))
+  return result
+
+
+def cherry_picked(base):
+  return set(CHERRY_PICK.findall(git('log', '--format=%B', f'{base}..HEAD')))
+
+
+def age_text(seconds):
+  hours = seconds / 3600
+  return f'{hours / 24:.1f} days' if hours >= 48 else f'{hours:.0f} hours'
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description=__doc__.split('\n\n')[0],
+      formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument('--upstream', default='upstream/main')
+  parser.add_argument('--max-age-hours', type=float, default=48)
+  parser.add_argument('--enforce', action='store_true')
+  parser.add_argument('--override-reason', default='')
+  parser.add_argument('--notes')
+  parser.add_argument('--json')
+  args = parser.parse_args()
+
+  now = time.time()
+  base = git('merge-base', 'HEAD', args.upstream).strip()
+  base_sha, _, base_date, base_subject = commits('-1', base)[0]
+  missing = commits(args.upstream, '^HEAD')
+  security = [c for c in missing if SECURITY.search(c[3])]
+  picked = cherry_picked(base)
+  oldest = missing[-1] if missing else None
+  stale = bool(oldest) and now - oldest[1] > args.max_age_hours * 3600
+  override = args.override_reason.strip()
+  unpicked = [c for c in security if c[0] not in picked]
+
+  lines = [
+      '## Upstream PDFium',
+      '',
+      f'- Based on upstream `{base_sha[:9]}` ({base_date[:10]}) {base_subject}',
+  ]
+  if oldest:
+    lines.append(f'- Missing {len(missing)} upstream commits; the oldest landed '
+                 f'{age_text(now - oldest[1])} ago (limit: '
+                 f'{age_text(args.max_age_hours * 3600)}).')
+  else:
+    lines.append('- Missing no upstream commit.')
+  if stale and override:
+    lines.append(f'- **Gate overridden:** {override}')
+  if missing:
+    lines += ['', '### Missing upstream commits', '']
+    for commit in missing:
+      sha, _, date, subject = commit
+      mark = ''
+      if commit in security:
+        mark = (' (likely security fix; cherry-picked)'
+                if sha in picked else ' (**likely security fix**)')
+      lines.append(f'- `{sha[:9]}` {date[:10]} {subject}{mark}')
+  report = '\n'.join(lines) + '\n'
+
+  print(report)
+  if os.environ.get('GITHUB_STEP_SUMMARY'):
+    with open(os.environ['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') as f:
+      f.write(report)
+  if os.environ.get('GITHUB_ACTIONS'):
+    for sha, _, _, subject in security:
+      print(f'::warning title=Likely upstream security fix::{sha[:9]} {subject}')
+  if args.notes:
+    with open(args.notes, 'w', encoding='utf-8') as f:
+      f.write(report)
+  if args.json:
+    with open(args.json, 'w', encoding='utf-8') as f:
+      json.dump({'sha': base_sha, 'date': base_date, 'subject': base_subject,
+                 'missing': len(missing), 'override': override if stale else ''},
+                f, indent=2)
+      f.write('\n')
+
+  if not args.enforce or not stale:
+    return
+  if not override:
+    sys.exit(f'error: missing upstream commits older than '
+             f'{age_text(args.max_age_hours * 3600)}: sync first '
+             '(docs/embedpdf/upstream-sync.md), or pass an override reason.')
+  if unpicked:
+    sys.exit('error: an override still needs every likely security fix it '
+             'holds back, cherry-picked with -x: ' +
+             ', '.join(c[0][:9] for c in unpicked))
+  print('warning: release gate overridden: ' + override)
+
+
+if __name__ == '__main__':
+  main()

scripts/embedpdf-runtime/globals-allowlist.txt

@@ -46,6 +46,7 @@ set-once | (anonymous namespace)::AddStandardFont(CPDF_Document*)::encoding | Up
 set-once | guard variable for (anonymous namespace)::AddStandardFont(CPDF_Document*)::encoding | Its initialization guard.
 set-once | fxcodec::(anonymous namespace)::Basis()::basis | The JPEG writer's DCT basis: a const table built once (thread-safe static initialization) and only read after.
 set-once | guard variable for fxcodec::(anonymous namespace)::Basis()::basis | Its initialization guard.
+set-once | CStretchEngine::Expand1BppRow(*)::kBitsToBytes | Upstream: a table built at compile time (constinit) and only read; not const, so it sits in writable data.
 
 # --- Third-party code, shared by design.
 third-party | std::__Cr::* | libc++: locale and facet state behind its own locks.
@@ -77,7 +78,8 @@ third-party | known_tables | HarfBuzz subsetter: a constant table.
 third-party | langs | HarfBuzz: the language list, behind atomics.
 third-party | endchar_str | HarfBuzz CFF: a constant.
 third-party | only_cleanup_pass_is_decoded | openjpeg: a one-time warning flag, set under a mutex.
-third-party | Cr_z_arm_cpu_enable_* | zlib: CPU features, detected once.
+third-party | Cr_z_*_cpu_enable_* | zlib: CPU features, detected once (under cpu_check_inited_once).
+third-party | cpu_check_inited_once | zlib: the pthread_once flag of its CPU check.
 third-party | cpu_info_ | libyuv: CPU features, detected on first use; read and written with relaxed atomics, and every thread detects the same value.
 third-party | SkFibonacci47 | Skia path ops: a constant table.
 third-party | SkMatrix::I()::identity | Skia path ops: a constant.
@@ -86,3 +88,6 @@ third-party | SkPathData::PeekEmptySingleton()::gEmpty | Skia path ops: a once-b
 third-party | guard variable for SkPath*::Peek* | Skia path ops: initialization guards.
 third-party | next_pathdata_unique_id()::nextID | Skia path ops: an atomic counter.
 third-party | fast_float::detail::rounds_to_nearest()::fmin | fast_float: a volatile constant.
+third-party | __dso_handle | The C runtime (ELF): this library's handle for atexit, set when it loads.
+third-party | DW.ref.__gxx_personality_v0 | The C++ runtime (ELF): the exception personality's address, set when the library loads.
+third-party | completed.0 | The C runtime (ELF, crtstuff): set once, when the library's destructors run at unload.