fix(data-repo): runlog index marks damaged .jsonl.gz rows with an error field

scripts/build-runlog-index.py

@@ -15,7 +15,8 @@
 (logging_stopped, else last runner rx_ms), duration_s, complete
 (true = runner 'sequence-complete' seen, false = 'aborted', null = unknown),
 plus the run_metadata fields the catalog shows (protocol_filename, experimenter,
-genotype, sex, fly_number, age, notes, rig_id, timestamp_start).
+genotype, sex, fly_number, age, notes, rig_id, timestamp_start). A damaged `.gz`
+additionally carries `error` ('gzip truncated' | 'gzip unreadable').
 
 usage: build-runlog-index.py <clone-root> [--write] [--folder NAME ...]
        build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME ...]
@@ -99,15 +100,18 @@ def is_gz(name):
     return name.lower().endswith('.gz')
 
 def inflate(raw):
-    """gzip bytes → text. Tolerates a truncated/corrupt trailer (a half-written
-    upload) by falling back to a streaming decompressor that keeps what it got."""
+    """gzip bytes → (text, error). error is None for a clean stream; 'gzip truncated'
+    when only a prefix could be recovered (a half-written upload — the head is still
+    indexed, the end state is unknown); 'gzip unreadable' when nothing could. The
+    error is carried into the index row so a damaged upload is visible as damaged,
+    not as a merely unfinished run."""
     try:
-        return gzip.decompress(raw).decode('utf-8', 'replace')
+        return gzip.decompress(raw).decode('utf-8', 'replace'), None
     except Exception:
         d = zlib.decompressobj(16 + zlib.MAX_WBITS)
         try: out = d.decompress(raw)
         except zlib.error: out = b''
-        return out.decode('utf-8', 'replace')
+        return out.decode('utf-8', 'replace'), ('gzip truncated' if out else 'gzip unreadable')
 
 def head_tail(text):
     return text[:HEAD_BYTES], text[-TAIL_BYTES:]
@@ -133,11 +137,12 @@ def is_runlog_name(name):
     n = name.lower()
     return n.endswith('.jsonl') or n.endswith('.jsonl.gz')
 
-def bookends(path, size=None, head=None, tail=None):
+def bookends(path, size=None, head=None, tail=None, error=None):
     if head is None:
         size = os.path.getsize(path)
         if is_gz(path):
-            with open(path, 'rb') as fh: head, tail = head_tail(inflate(fh.read()))
+            with open(path, 'rb') as fh: text, error = inflate(fh.read())
+            head, tail = head_tail(text)
         else:
             with open(path, 'rb') as fh:
                 head = fh.read(HEAD_BYTES).decode('utf-8', 'replace')
@@ -164,6 +169,9 @@ def recs(text):
     keep = ['run_id', 'rig_id', 'protocol_filename', 'protocol_sha256', 'experimenter', 'genotype', 'sex', 'fly_number', 'age', 'notes', 'timestamp_start', 'tool_version']
     entry = {k: meta.get(k) for k in keep if k in meta}
     entry.update({'file': os.path.basename(path), 'size': size, 'started_ms': started, 'stopped_ms': stopped, 'duration_s': dur, 'complete': complete})
+    if error:
+        entry['error'] = error  # only ever present for a damaged .gz — plain-file rows are unchanged
+        print(f'  WARNING {path}: {error}', file=sys.stderr)
     return entry
 
 def main_github(repo, branch, write, only):
@@ -175,12 +183,14 @@ def main_github(repo, branch, write, only):
         items = [i for i in _gh_api(repo, f"contents/{d['path']}?ref={branch}") if i['type'] == 'file' and is_runlog_name(i['name'])]
         runs = []
         for it in items:
+            err = None
             if is_gz(it['name']):
-                head, tail = head_tail(inflate(_raw_full(repo, branch, it['path'])))
+                text, err = inflate(_raw_full(repo, branch, it['path']))
+                head, tail = head_tail(text)
             else:
                 head = _raw_range(repo, branch, it['path'], f'bytes=0-{HEAD_BYTES-1}')
                 tail = _raw_range(repo, branch, it['path'], f'bytes=-{TAIL_BYTES}') if it['size'] > TAIL_BYTES else head
-            runs.append(bookends(it['path'], it['size'], head, tail))
+            runs.append(bookends(it['path'], it['size'], head, tail, err))
         index = {'format_version': 1, 'folder': name, 'generated': 'scripts/build-runlog-index.py', 'runs': runs}
         total += len(runs)
         known = sum(1 for r in runs if r['duration_s'] is not None); aborted = sum(1 for r in runs if r['complete'] is False)

tests/test-build-runlog-index.py

@@ -91,6 +91,15 @@ def synthetic_log(n_frames=3000, complete=True):
     e3 = M.bookends(trunc)
     check(e3['run_id'] == 'r2' and e3['started_ms'] == 1788636439304, 'truncated gz: metadata + start still read')
     check(e3['stopped_ms'] is None and e3['duration_s'] is None and e3['complete'] is None, 'truncated gz: no invented end state')
+    check(e3.get('error') == 'gzip truncated', 'truncated gz: row is marked damaged, not merely unfinished')
+    check('error' not in e1 and 'error' not in e2, 'plain and clean-gz rows carry no error key (index unchanged for them)')
+
+    print('-- unreadable gz (not gzip at all, named .jsonl.gz) --')
+    junk = os.path.join(folder, 'p3__michael__2026-09-05T19-27-19__r5.jsonl.gz')
+    with open(junk, 'wb') as fh:
+        fh.write(b'this is not a gzip stream' * 100)
+    e5 = M.bookends(junk)
+    check(e5.get('error') == 'gzip unreadable' and 'run_id' not in e5, 'unreadable gz: marked, no metadata invented, never raises')
 
     print('-- aborted run without logging_stopped --')
     ab = os.path.join(folder, 'p3__michael__2026-09-05T19-27-19__r4.jsonl.gz')
@@ -104,7 +113,9 @@ def synthetic_log(n_frames=3000, complete=True):
     check(out.returncode == 0, 'CLI exit 0: ' + out.stdout.strip().splitlines()[-1])
     idx = json.load(open(os.path.join(folder, 'index.json')))
     files = sorted(r['file'] for r in idx['runs'])
-    check(files == sorted(os.path.basename(p) for p in (plain, gz, trunc, ab)), f'index lists all 4 files: {files}')
+    check(files == sorted(os.path.basename(p) for p in (plain, gz, trunc, ab, junk)), f'index lists all 5 files: {files}')
+    errs = {r['file']: r.get('error') for r in idx['runs']}
+    check(errs[os.path.basename(trunc)] == 'gzip truncated' and errs[os.path.basename(junk)] == 'gzip unreadable' and errs[os.path.basename(plain)] is None, 'errors land in index.json for the damaged files only')
     check(idx['format_version'] == 1 and idx['folder'] == 'rig9', 'index envelope unchanged')
 
     print('-- name filter for --github mode --')