diff --git a/dashboard/data-browser/README.md b/dashboard/data-browser/README.md index 9fa407ec..81de4831 100644 --- a/dashboard/data-browser/README.md +++ b/dashboard/data-browser/README.md @@ -45,9 +45,11 @@ runs; for GitHub-indexed runs that have not been loaded it comes from the folder file was tried first, but browsers cannot do it: raw.githubusercontent.com refuses the CORS preflight; the GitHub API ignores `Range` entirely.) Build or refresh the index with `scripts/build-runlog-index.py --github owner/repo --write` (reads only -a 64 KB head + 4 KB tail per file); Arena Studio will append to it after each -auto-committed run (planned with the behavior_v2 work). Runs missing from the index -show "—" until it is refreshed. Aborted runs are flagged ⚠ with +a 64 KB head + 4 KB tail per file). In the data repos this runs automatically: the +`runlog-index` GitHub Action (template in `scripts/data-repo-workflows/`) rebuilds a +folder's index on every push under `runlogs/`, so a run shows its duration about a +minute after it lands. Arena Studio does not write the index. Runs missing from an +index show "—" until the Action has run. Aborted runs are flagged ⚠ with the tooltip naming the end state. Optional columns: age, experimenter, file size. ## Analysis pages diff --git a/dashboard/data-browser/app.js b/dashboard/data-browser/app.js index 2ed2cbd1..460aa2e8 100644 --- a/dashboard/data-browser/app.js +++ b/dashboard/data-browser/app.js @@ -874,7 +874,8 @@ async function browseGithub() { ); const descriptor = A.parseMetadataPrefix(prefix, item.name, item.path); // Start / duration / end state come from the folder's index.json - // (scripts/build-runlog-index.py; Studio append planned). A tail Range + // (scripts/build-runlog-index.py, run by the data repo's runlog-index + // GitHub Action on every push under runlogs/). A tail Range // read was tried first but the browser's CORS preflight is refused by // raw.githubusercontent.com — G.fetchSuffix stays for non-browser use. const indexed = diff --git a/scripts/build-runlog-index.py b/scripts/build-runlog-index.py index d2dd0396..f3c932d9 100755 --- a/scripts/build-runlog-index.py +++ b/scripts/build-runlog-index.py @@ -5,8 +5,11 @@ state WITHOUT downloading multi-MB logs. Reading a file tail from the browser is blocked (raw.githubusercontent.com rejects the CORS preflight that a `Range` header triggers), so every runlog folder carries a small index the dashboard -fetches in ONE request. Arena Studio appends to it after each auto-commit; this -script backfills/refreshes it from the files on disk (idempotent — re-run any time). +fetches in ONE request. The index has ONE writer: this script, run by the data +repo's `runlog-index` GitHub Action on every push under runlogs/ (see +scripts/data-repo-workflows/runlog-index.yml) — and by hand for a backfill. +Idempotent: each folder's index is rebuilt from the files present, so deletions +and out-of-band pushes (git push of a large log, migrations) are picked up too. Per run: run_id, file, size, started_ms (logging_started), stopped_ms (logging_stopped, else last runner rx_ms), duration_s, complete @@ -14,14 +17,15 @@ plus the run_metadata fields the catalog shows (protocol_filename, experimenter, genotype, sex, fly_number, age, notes, rig_id, timestamp_start). -usage: build-runlog-index.py [--write] [--folder NAME] - build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME] +usage: build-runlog-index.py [--write] [--folder NAME ...] + build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME ...] + --folder may repeat (the Action passes only the folders a push touched). (default = dry run: prints the table; --write rewrites each index.json — on disk for a clone, via the Contents API for --github. --github reads only a 64 KB head + 4 KB tail per file with Range requests on the raw URL, so it never downloads the logs; needs `gh auth token` or $GITHUB_TOKEN.) """ -import json, os, sys, glob, subprocess, urllib.request, base64 +import json, os, sys, glob, subprocess, urllib.request, urllib.error, base64 HEAD_BYTES = 65536 TAIL_BYTES = 4096 @@ -45,13 +49,45 @@ def _gh_api(repo, path, method='GET', body=None): st, raw, _ = _http(f'https://api.github.com/repos/{repo}/{path}', h, data, method) return json.loads(raw) if raw else None +def _put_index(repo, branch, path, content, name, n): + """PUT index.json unless it is already identical (no no-op commits); on a + stale-sha conflict (409/422 — another run wrote the same file meanwhile) + re-read the sha and retry, up to 3 attempts.""" + import urllib.error, time + for attempt in range(3): + sha = None + try: + cur = _gh_api(repo, f'contents/{path}?ref={branch}') + sha = cur.get('sha') + if cur.get('encoding') == 'base64' and base64.b64decode(cur.get('content') or '').decode('utf-8', 'replace') == content: + print(f' {name}: index.json unchanged — skipped'); return None + except Exception: pass + body = {'message': f'runlogs({name}): refresh index.json ({n} runs)', 'content': base64.b64encode(content.encode()).decode(), 'branch': branch} + if sha: body['sha'] = sha + try: + return _gh_api(repo, f'contents/{path}', 'PUT', body) + except urllib.error.HTTPError as e: + if e.code in (409, 422) and attempt < 2: + time.sleep(2 + attempt); continue + raise + def _raw_range(repo, branch, path, rng): + """Partial read of one file via raw.githubusercontent.com (honours Range; the + API host does not). Retries transient network errors (connection resets were + seen on a 340-request full rebuild).""" + import time tok = _token(); h = {'Range': rng} if tok: h['Authorization'] = 'Bearer ' + tok url = f'https://raw.githubusercontent.com/{repo}/{branch}/{urllib.request.quote(path)}' - st, raw, hdr = _http(url, h) - if st != 206: raise RuntimeError(f'{path}: expected 206 for {rng}, got {st}') - return raw.decode('utf-8', 'replace') + last = None + for attempt in range(4): + try: + st, raw, hdr = _http(url, h) + if st != 206: raise RuntimeError(f'{path}: expected 206 for {rng}, got {st}') + return raw.decode('utf-8', 'replace') + except (ConnectionError, OSError, urllib.error.URLError) as e: # incl. ConnectionResetError + last = e; time.sleep(1.5 * (attempt + 1)) + raise RuntimeError(f'{path}: {last}') def bookends(path, size=None, head=None, tail=None): if head is None: @@ -88,7 +124,7 @@ def main_github(repo, branch, write, only): total = 0 for d in sorted(dirs, key=lambda x: x['name']): name = d['name'] - if only and name != only: continue + if only and name not in only: continue items = [i for i in _gh_api(repo, f"contents/{d['path']}?ref={branch}") if i['type'] == 'file' and i['name'].endswith('.jsonl')] runs = [] for it in items: @@ -100,20 +136,13 @@ def main_github(repo, branch, write, only): known = sum(1 for r in runs if r['duration_s'] is not None); aborted = sum(1 for r in runs if r['complete'] is False) print(f"{name:12s} {len(runs):3d} runs duration known {known:3d} aborted {aborted:2d}") if write: - path = f"{d['path']}/index.json" - content = json.dumps(index, indent=1) + '\n' - sha = None - try: sha = _gh_api(repo, f'contents/{path}?ref={branch}').get('sha') - except Exception: pass - body = {'message': f'runlogs({name}): refresh index.json ({len(runs)} runs)', 'content': base64.b64encode(content.encode()).decode(), 'branch': branch} - if sha: body['sha'] = sha - _gh_api(repo, f'contents/{path}', 'PUT', body) + _put_index(repo, branch, f"{d['path']}/index.json", json.dumps(index, indent=1) + '\n', name, len(runs)) print(f"{'WROTE' if write else 'dry-run'}: {total} runs in {len(dirs)} folders ({repo}@{branch})") def main(): args = [a for a in sys.argv[1:] if not a.startswith('--')] write = '--write' in sys.argv - only = sys.argv[sys.argv.index('--folder') + 1] if '--folder' in sys.argv else None + only = {sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == '--folder' and i + 1 < len(sys.argv)} or None if '--github' in sys.argv: repo = sys.argv[sys.argv.index('--github') + 1] branch = sys.argv[sys.argv.index('--branch') + 1] if '--branch' in sys.argv else 'main' @@ -124,7 +153,7 @@ def main(): total = 0 for folder in folders: name = os.path.basename(folder) - if only and name != only: continue + if only and name not in only: continue files = sorted(glob.glob(os.path.join(folder, '*.jsonl')) + glob.glob(os.path.join(folder, '*.jsonl.gz'))) files = [f for f in files if not f.endswith('.gz')] # gz handled once behavior_v2 lands runs = [bookends(f) for f in files] diff --git a/scripts/data-repo-workflows/runlog-index.yml b/scripts/data-repo-workflows/runlog-index.yml new file mode 100644 index 00000000..fd7a179f --- /dev/null +++ b/scripts/data-repo-workflows/runlog-index.yml @@ -0,0 +1,64 @@ +# runlog-index — keep runlogs//index.json current. +# +# Installed into every Arena data repo by webDisplayTools/scripts/seed-data-repo.sh +# (source of truth: webDisplayTools/scripts/data-repo-workflows/runlog-index.yml; +# the script it runs is a copy of webDisplayTools/scripts/build-runlog-index.py). +# +# Why: the data-browser dashboard shows each run's start / duration / end state +# from this index, because a browser cannot Range-read a file tail from GitHub +# (CORS preflight 403). The index has ONE writer — this workflow — and is rebuilt +# per folder from the files actually present, so Arena Studio auto-commits, +# hand-pushed large logs, migrations and deletions are all picked up. +# +# Loop guard: the bot's own commits touch only index.json, which the `paths` +# filter excludes. Concurrency: runs queue instead of racing. +name: runlog index +on: + push: + branches: [main] + paths: + - 'runlogs/**' + - '!runlogs/**/index.json' + workflow_dispatch: {} # rebuild every folder by hand +permissions: + contents: write +concurrency: + group: runlog-index + cancel-in-progress: false +jobs: + index: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: .github # never pull the multi-GB logs + sparse-checkout-cone-mode: true + - name: Folders touched by this push + id: touched + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + EVENT: ${{ github.event_name }} + run: | + # Changed files from the compare API (independent of push-payload shape / + # truncation). workflow_dispatch → no range → rebuild every folder. + if [ "$EVENT" = "push" ] && [ -n "$BEFORE" ] && ! echo "$BEFORE" | grep -q '^0*$'; then + files=$(gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$AFTER" --paginate --jq '.files[].filename' || true) + else + files="" + fi + folders=$(printf '%s\n' "$files" | awk -F/ '$1=="runlogs" && NF>=3 && $NF!="index.json" {print $2}' | sort -u) + if [ -n "$folders" ]; then + args=$(printf -- '--folder %s ' $folders); list=$(echo $folders | tr ' ' ',') + else + args=""; list="ALL" + fi + echo "args=$args" >> "$GITHUB_OUTPUT"; echo "list=$list" >> "$GITHUB_OUTPUT" + echo "changed files: $(printf '%s\n' "$files" | grep -c . || true) → folders: $list" + - name: Rebuild index.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Folders: ${{ steps.touched.outputs.list }}" + python3 .github/scripts/build-runlog-index.py --github "$GITHUB_REPOSITORY" --branch main --write ${{ steps.touched.outputs.args }}