diff --git a/scripts/build-runlog-index.py b/scripts/build-runlog-index.py index fc5dad05..81c60269 100755 --- a/scripts/build-runlog-index.py +++ b/scripts/build-runlog-index.py @@ -15,7 +15,8 @@ (logging_stopped, else last runner rx_ms), duration_s, complete (true = runner 'sequence-complete' seen, false = 'aborted', null = unknown), plus the run_metadata fields the catalog shows (protocol_filename, experimenter, -genotype, sex, fly_number, age, notes, rig_id, timestamp_start). +genotype, sex, fly_number, age, notes, rig_id, timestamp_start). A damaged `.gz` +additionally carries `error` ('gzip truncated' | 'gzip unreadable'). usage: build-runlog-index.py [--write] [--folder NAME ...] build-runlog-index.py --github owner/repo [--branch main] [--write] [--folder NAME ...] @@ -99,15 +100,18 @@ def is_gz(name): return name.lower().endswith('.gz') def inflate(raw): - """gzip bytes → text. Tolerates a truncated/corrupt trailer (a half-written - upload) by falling back to a streaming decompressor that keeps what it got.""" + """gzip bytes → (text, error). error is None for a clean stream; 'gzip truncated' + when only a prefix could be recovered (a half-written upload — the head is still + indexed, the end state is unknown); 'gzip unreadable' when nothing could. The + error is carried into the index row so a damaged upload is visible as damaged, + not as a merely unfinished run.""" try: - return gzip.decompress(raw).decode('utf-8', 'replace') + return gzip.decompress(raw).decode('utf-8', 'replace'), None except Exception: d = zlib.decompressobj(16 + zlib.MAX_WBITS) try: out = d.decompress(raw) except zlib.error: out = b'' - return out.decode('utf-8', 'replace') + return out.decode('utf-8', 'replace'), ('gzip truncated' if out else 'gzip unreadable') def head_tail(text): return text[:HEAD_BYTES], text[-TAIL_BYTES:] @@ -133,11 +137,12 @@ def is_runlog_name(name): n = name.lower() return n.endswith('.jsonl') or n.endswith('.jsonl.gz') -def bookends(path, size=None, head=None, tail=None): +def bookends(path, size=None, head=None, tail=None, error=None): if head is None: size = os.path.getsize(path) if is_gz(path): - with open(path, 'rb') as fh: head, tail = head_tail(inflate(fh.read())) + with open(path, 'rb') as fh: text, error = inflate(fh.read()) + head, tail = head_tail(text) else: with open(path, 'rb') as fh: head = fh.read(HEAD_BYTES).decode('utf-8', 'replace') @@ -164,6 +169,9 @@ def recs(text): keep = ['run_id', 'rig_id', 'protocol_filename', 'protocol_sha256', 'experimenter', 'genotype', 'sex', 'fly_number', 'age', 'notes', 'timestamp_start', 'tool_version'] entry = {k: meta.get(k) for k in keep if k in meta} entry.update({'file': os.path.basename(path), 'size': size, 'started_ms': started, 'stopped_ms': stopped, 'duration_s': dur, 'complete': complete}) + if error: + entry['error'] = error # only ever present for a damaged .gz — plain-file rows are unchanged + print(f' WARNING {path}: {error}', file=sys.stderr) return entry def main_github(repo, branch, write, only): @@ -175,12 +183,14 @@ def main_github(repo, branch, write, only): items = [i for i in _gh_api(repo, f"contents/{d['path']}?ref={branch}") if i['type'] == 'file' and is_runlog_name(i['name'])] runs = [] for it in items: + err = None if is_gz(it['name']): - head, tail = head_tail(inflate(_raw_full(repo, branch, it['path']))) + text, err = inflate(_raw_full(repo, branch, it['path'])) + head, tail = head_tail(text) else: head = _raw_range(repo, branch, it['path'], f'bytes=0-{HEAD_BYTES-1}') tail = _raw_range(repo, branch, it['path'], f'bytes=-{TAIL_BYTES}') if it['size'] > TAIL_BYTES else head - runs.append(bookends(it['path'], it['size'], head, tail)) + runs.append(bookends(it['path'], it['size'], head, tail, err)) index = {'format_version': 1, 'folder': name, 'generated': 'scripts/build-runlog-index.py', 'runs': runs} total += len(runs) known = sum(1 for r in runs if r['duration_s'] is not None); aborted = sum(1 for r in runs if r['complete'] is False) diff --git a/tests/test-build-runlog-index.py b/tests/test-build-runlog-index.py index 3bd5b900..057df183 100644 --- a/tests/test-build-runlog-index.py +++ b/tests/test-build-runlog-index.py @@ -91,6 +91,15 @@ def synthetic_log(n_frames=3000, complete=True): e3 = M.bookends(trunc) check(e3['run_id'] == 'r2' and e3['started_ms'] == 1788636439304, 'truncated gz: metadata + start still read') check(e3['stopped_ms'] is None and e3['duration_s'] is None and e3['complete'] is None, 'truncated gz: no invented end state') + check(e3.get('error') == 'gzip truncated', 'truncated gz: row is marked damaged, not merely unfinished') + check('error' not in e1 and 'error' not in e2, 'plain and clean-gz rows carry no error key (index unchanged for them)') + + print('-- unreadable gz (not gzip at all, named .jsonl.gz) --') + junk = os.path.join(folder, 'p3__michael__2026-09-05T19-27-19__r5.jsonl.gz') + with open(junk, 'wb') as fh: + fh.write(b'this is not a gzip stream' * 100) + e5 = M.bookends(junk) + check(e5.get('error') == 'gzip unreadable' and 'run_id' not in e5, 'unreadable gz: marked, no metadata invented, never raises') print('-- aborted run without logging_stopped --') ab = os.path.join(folder, 'p3__michael__2026-09-05T19-27-19__r4.jsonl.gz') @@ -104,7 +113,9 @@ def synthetic_log(n_frames=3000, complete=True): check(out.returncode == 0, 'CLI exit 0: ' + out.stdout.strip().splitlines()[-1]) idx = json.load(open(os.path.join(folder, 'index.json'))) files = sorted(r['file'] for r in idx['runs']) - check(files == sorted(os.path.basename(p) for p in (plain, gz, trunc, ab)), f'index lists all 4 files: {files}') + check(files == sorted(os.path.basename(p) for p in (plain, gz, trunc, ab, junk)), f'index lists all 5 files: {files}') + errs = {r['file']: r.get('error') for r in idx['runs']} + check(errs[os.path.basename(trunc)] == 'gzip truncated' and errs[os.path.basename(junk)] == 'gzip unreadable' and errs[os.path.basename(plain)] is None, 'errors land in index.json for the damaged files only') check(idx['format_version'] == 1 and idx['folder'] == 'rig9', 'index envelope unchanged') print('-- name filter for --github mode --')