feat(data-repo): runlog-index GitHub Action keeps runlogs/<folder>/index.json current

dashboard/data-browser/README.md

@@ -45,9 +45,11 @@ runs; for GitHub-indexed runs that have not been loaded it comes from the folder
 file was tried first, but browsers cannot do it: raw.githubusercontent.com refuses
 the CORS preflight; the GitHub API ignores `Range` entirely.) Build or refresh the
 index with `scripts/build-runlog-index.py --github owner/repo --write` (reads only
-a 64 KB head + 4 KB tail per file); Arena Studio will append to it after each
-auto-committed run (planned with the behavior_v2 work). Runs missing from the index
-show "—" until it is refreshed. Aborted runs are flagged ⚠ with
+a 64 KB head + 4 KB tail per file). In the data repos this runs automatically: the
+`runlog-index` GitHub Action (template in `scripts/data-repo-workflows/`) rebuilds a
+folder's index on every push under `runlogs/`, so a run shows its duration about a
+minute after it lands. Arena Studio does not write the index. Runs missing from an
+index show "—" until the Action has run. Aborted runs are flagged ⚠ with
 the tooltip naming the end state. Optional columns: age, experimenter, file size.
 
 ## Analysis pages

dashboard/data-browser/app.js

@@ -874,7 +874,8 @@ async function browseGithub() {
                 );
                 const descriptor = A.parseMetadataPrefix(prefix, item.name, item.path);
                 // Start / duration / end state come from the folder's index.json
-                // (scripts/build-runlog-index.py; Studio append planned). A tail Range
+                // (scripts/build-runlog-index.py, run by the data repo's runlog-index
+                // GitHub Action on every push under runlogs/). A tail Range
                 // read was tried first but the browser's CORS preflight is refused by
                 // raw.githubusercontent.com — G.fetchSuffix stays for non-browser use.
                 const indexed =

scripts/build-runlog-index.py

@@ -5,23 +5,27 @@
 state WITHOUT downloading multi-MB logs. Reading a file tail from the browser is
 blocked (raw.githubusercontent.com rejects the CORS preflight that a `Range`
 header triggers), so every runlog folder carries a small index the dashboard
-fetches in ONE request. Arena Studio appends to it after each auto-commit; this
-script backfills/refreshes it from the files on disk (idempotent — re-run any time).
+fetches in ONE request. The index has ONE writer: this script, run by the data
+repo's `runlog-index` GitHub Action on every push under runlogs/ (see
+scripts/data-repo-workflows/runlog-index.yml) — and by hand for a backfill.
+Idempotent: each folder's index is rebuilt from the files present, so deletions
+and out-of-band pushes (git push of a large log, migrations) are picked up too.
 
 Per run: run_id, file, size, started_ms (logging_started), stopped_ms
 (logging_stopped, else last runner rx_ms), duration_s, complete
 (true = runner 'sequence-complete' seen, false = 'aborted', null = unknown),
 plus the run_metadata fields the catalog shows (protocol_filename, experimenter,
 genotype, sex, fly_number, age, notes, rig_id, timestamp_start).
 
-usage: build-runlog-index.py <clone-root> [--write] [--folder NAME]
-       build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME]
+usage: build-runlog-index.py <clone-root> [--write] [--folder NAME ...]
+       build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME ...]
+  --folder may repeat (the Action passes only the folders a push touched).
   (default = dry run: prints the table; --write rewrites each index.json —
    on disk for a clone, via the Contents API for --github. --github reads only
    a 64 KB head + 4 KB tail per file with Range requests on the raw URL, so it
    never downloads the logs; needs `gh auth token` or $GITHUB_TOKEN.)
 """
-import json, os, sys, glob, subprocess, urllib.request, base64
+import json, os, sys, glob, subprocess, urllib.request, urllib.error, base64
 
 HEAD_BYTES = 65536
 TAIL_BYTES = 4096
@@ -45,13 +49,45 @@ def _gh_api(repo, path, method='GET', body=None):
     st, raw, _ = _http(f'https://api.github.com/repos/{repo}/{path}', h, data, method)
     return json.loads(raw) if raw else None
 
+def _put_index(repo, branch, path, content, name, n):
+    """PUT index.json unless it is already identical (no no-op commits); on a
+    stale-sha conflict (409/422 — another run wrote the same file meanwhile)
+    re-read the sha and retry, up to 3 attempts."""
+    import urllib.error, time
+    for attempt in range(3):
+        sha = None
+        try:
+            cur = _gh_api(repo, f'contents/{path}?ref={branch}')
+            sha = cur.get('sha')
+            if cur.get('encoding') == 'base64' and base64.b64decode(cur.get('content') or '').decode('utf-8', 'replace') == content:
+                print(f'  {name}: index.json unchanged — skipped'); return None
+        except Exception: pass
+        body = {'message': f'runlogs({name}): refresh index.json ({n} runs)', 'content': base64.b64encode(content.encode()).decode(), 'branch': branch}
+        if sha: body['sha'] = sha
+        try:
+            return _gh_api(repo, f'contents/{path}', 'PUT', body)
+        except urllib.error.HTTPError as e:
+            if e.code in (409, 422) and attempt < 2:
+                time.sleep(2 + attempt); continue
+            raise
+
 def _raw_range(repo, branch, path, rng):
+    """Partial read of one file via raw.githubusercontent.com (honours Range; the
+    API host does not). Retries transient network errors (connection resets were
+    seen on a 340-request full rebuild)."""
+    import time
     tok = _token(); h = {'Range': rng}
     if tok: h['Authorization'] = 'Bearer ' + tok
     url = f'https://raw.githubusercontent.com/{repo}/{branch}/{urllib.request.quote(path)}'
-    st, raw, hdr = _http(url, h)
-    if st != 206: raise RuntimeError(f'{path}: expected 206 for {rng}, got {st}')
-    return raw.decode('utf-8', 'replace')
+    last = None
+    for attempt in range(4):
+        try:
+            st, raw, hdr = _http(url, h)
+            if st != 206: raise RuntimeError(f'{path}: expected 206 for {rng}, got {st}')
+            return raw.decode('utf-8', 'replace')
+        except (ConnectionError, OSError, urllib.error.URLError) as e:  # incl. ConnectionResetError
+            last = e; time.sleep(1.5 * (attempt + 1))
+    raise RuntimeError(f'{path}: {last}')
 
 def bookends(path, size=None, head=None, tail=None):
     if head is None:
@@ -88,7 +124,7 @@ def main_github(repo, branch, write, only):
     total = 0
     for d in sorted(dirs, key=lambda x: x['name']):
         name = d['name']
-        if only and name != only: continue
+        if only and name not in only: continue
         items = [i for i in _gh_api(repo, f"contents/{d['path']}?ref={branch}") if i['type'] == 'file' and i['name'].endswith('.jsonl')]
         runs = []
         for it in items:
@@ -100,20 +136,13 @@ def main_github(repo, branch, write, only):
         known = sum(1 for r in runs if r['duration_s'] is not None); aborted = sum(1 for r in runs if r['complete'] is False)
         print(f"{name:12s} {len(runs):3d} runs  duration known {known:3d}  aborted {aborted:2d}")
         if write:
-            path = f"{d['path']}/index.json"
-            content = json.dumps(index, indent=1) + '\n'
-            sha = None
-            try: sha = _gh_api(repo, f'contents/{path}?ref={branch}').get('sha')
-            except Exception: pass
-            body = {'message': f'runlogs({name}): refresh index.json ({len(runs)} runs)', 'content': base64.b64encode(content.encode()).decode(), 'branch': branch}
-            if sha: body['sha'] = sha
-            _gh_api(repo, f'contents/{path}', 'PUT', body)
+            _put_index(repo, branch, f"{d['path']}/index.json", json.dumps(index, indent=1) + '\n', name, len(runs))
     print(f"{'WROTE' if write else 'dry-run'}: {total} runs in {len(dirs)} folders ({repo}@{branch})")
 
 def main():
     args = [a for a in sys.argv[1:] if not a.startswith('--')]
     write = '--write' in sys.argv
-    only = sys.argv[sys.argv.index('--folder') + 1] if '--folder' in sys.argv else None
+    only = {sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == '--folder' and i + 1 < len(sys.argv)} or None
     if '--github' in sys.argv:
         repo = sys.argv[sys.argv.index('--github') + 1]
         branch = sys.argv[sys.argv.index('--branch') + 1] if '--branch' in sys.argv else 'main'
@@ -124,7 +153,7 @@ def main():
     total = 0
     for folder in folders:
         name = os.path.basename(folder)
-        if only and name != only: continue
+        if only and name not in only: continue
         files = sorted(glob.glob(os.path.join(folder, '*.jsonl')) + glob.glob(os.path.join(folder, '*.jsonl.gz')))
         files = [f for f in files if not f.endswith('.gz')]  # gz handled once behavior_v2 lands
         runs = [bookends(f) for f in files]

scripts/data-repo-workflows/runlog-index.yml

@@ -0,0 +1,64 @@
+# runlog-index — keep runlogs/<folder>/index.json current.
+#
+# Installed into every Arena data repo by webDisplayTools/scripts/seed-data-repo.sh
+# (source of truth: webDisplayTools/scripts/data-repo-workflows/runlog-index.yml;
+# the script it runs is a copy of webDisplayTools/scripts/build-runlog-index.py).
+#
+# Why: the data-browser dashboard shows each run's start / duration / end state
+# from this index, because a browser cannot Range-read a file tail from GitHub
+# (CORS preflight 403). The index has ONE writer — this workflow — and is rebuilt
+# per folder from the files actually present, so Arena Studio auto-commits,
+# hand-pushed large logs, migrations and deletions are all picked up.
+#
+# Loop guard: the bot's own commits touch only index.json, which the `paths`
+# filter excludes. Concurrency: runs queue instead of racing.
+name: runlog index
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'runlogs/**'
+      - '!runlogs/**/index.json'
+  workflow_dispatch: {}   # rebuild every folder by hand
+permissions:
+  contents: write
+concurrency:
+  group: runlog-index
+  cancel-in-progress: false
+jobs:
+  index:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github   # never pull the multi-GB logs
+          sparse-checkout-cone-mode: true
+      - name: Folders touched by this push
+        id: touched
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BEFORE: ${{ github.event.before }}
+          AFTER: ${{ github.sha }}
+          EVENT: ${{ github.event_name }}
+        run: |
+          # Changed files from the compare API (independent of push-payload shape /
+          # truncation). workflow_dispatch → no range → rebuild every folder.
+          if [ "$EVENT" = "push" ] && [ -n "$BEFORE" ] && ! echo "$BEFORE" | grep -q '^0*$'; then
+            files=$(gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$AFTER" --paginate --jq '.files[].filename' || true)
+          else
+            files=""
+          fi
+          folders=$(printf '%s\n' "$files" | awk -F/ '$1=="runlogs" && NF>=3 && $NF!="index.json" {print $2}' | sort -u)
+          if [ -n "$folders" ]; then
+            args=$(printf -- '--folder %s ' $folders); list=$(echo $folders | tr ' ' ',')
+          else
+            args=""; list="ALL"
+          fi
+          echo "args=$args" >> "$GITHUB_OUTPUT"; echo "list=$list" >> "$GITHUB_OUTPUT"
+          echo "changed files: $(printf '%s\n' "$files" | grep -c . || true) → folders: $list"
+      - name: Rebuild index.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "Folders: ${{ steps.touched.outputs.list }}"
+          python3 .github/scripts/build-runlog-index.py --github "$GITHUB_REPOSITORY" --branch main --write ${{ steps.touched.outputs.args }}