repo-list, compact: handle archives with a missing metadata object, fixes #10435

src/borg/archive.py

@@ -668,7 +668,7 @@ def __init__(
                 else:
                     info = self.manifest.archives.get(name)
                 if info is None:
-                    raise self.DoesNotExist(name)
+                    raise self.DoesNotExist(bin_to_hex(name) if name_is_id else name)
             self.load(info.id, metadata=info.metadata)
 
     def _load_meta(self, id):
@@ -2805,7 +2805,11 @@ def robust_item_ids():
                 archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
                 try:
                     formatted = formatter.format_item(info, jsonline=False)
-                except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase):
+                    # the formatter uses defaults for keys like {comment} if it has no archive metadata.
+                    metadata_missing = bool(formatter.used_call_keys) and formatter.archive is None
+                except (Repository.ObjectNotFound, IntegrityErrorBase):
+                    metadata_missing = True
+                if metadata_missing:
                     # keys like {comment} need the archive metadata, which is damaged or missing here.
                     # use the values from the archive directory entry, they are always available.
                     formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}"

src/borg/archiver/compact_cmd.py

@@ -249,7 +249,7 @@ def mark_soft_deleted_used(self):
                         f"Soft-deleted archive {name} {hex_id}: {missing} objects missing from the index; "
                         f'"borg undelete" may not fully recover it.'
                     )
-            except (Repository.ObjectNotFound, IntegrityError) as e:
+            except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityError) as e:
                 logger.warning(f"Soft-deleted archive {name} {hex_id} cannot be fully preserved: {e}")
 
     def compact_packs(self):

src/borg/helpers/parseformat.py

@@ -1136,6 +1136,7 @@ def __init__(self, format, repository, manifest, key, *, deleted=False):
         self.id = None
         self.archive_info = None
         self._archive = None
+        self._archive_id = None  # the id self._archive was loaded for (self._archive can be None)
         self.deleted = deleted  # True if we want to deal with deleted archives.
         self.format_keys = {f[1] for f in Formatter().parse(format)}
         self.call_keys = {
@@ -1175,28 +1176,38 @@ def get_item_data(self, archive_info, jsonline=False):
 
     @property
     def archive(self):
-        """lazy load / update loaded archive"""
-        if self._archive is None or self._archive.id != self.id:
+        """lazy load / update loaded archive, None if the archive has no valid metadata object"""
+        if self._archive_id != self.id:
             from ..archive import Archive
 
             # the ArchiveInfo usually carries the archive's metadata, so this does not need to load it again.
-            self._archive = Archive(self.manifest, self.archive_info, deleted=self.deleted)
+            try:
+                self._archive = Archive(self.manifest, self.archive_info, deleted=self.deleted)
+            except Archive.DoesNotExist:
+                # the archives directory lists it, but its metadata object is missing or invalid, so we only
+                # have the placeholder values of the ArchiveInfo (see Archives._parse_archive_meta).
+                self._archive = None
+            self._archive_id = self.id
         return self._archive
 
     def get_meta(self, key, default=None):
-        return self.archive.metadata.get(key, default)
+        archive = self.archive
+        return archive.metadata.get(key, default) if archive is not None else default
 
     def get_ts_start(self):
-        return self.format_time(self.archive.ts_start)
+        archive = self.archive
+        return self.format_time(archive.ts_start if archive is not None else self.archive_info.ts)
 
     def get_ts_end(self):
-        return self.format_time(self.archive.ts_end)
+        archive = self.archive
+        return self.format_time(archive.ts_end if archive is not None else self.archive_info.ts)
 
     def format_time(self, ts):
         return OutputTimestamp(ts)
 
     def get_tags(self):
-        return ",".join(sorted(self.archive.tags))
+        archive = self.archive
+        return ",".join(sorted(archive.tags if archive is not None else self.archive_info.tags))
 
 
 class ItemFormatter(BaseFormatter):

src/borg/testsuite/archiver/compact_cmd_test.py

@@ -445,6 +445,36 @@ def test_compact_keeps_undelete_data_when_chunks_missing(archivers, request):
         assert fd.read() == b"G" * (1024 * 80)  # its data survived compaction
 
 
+def test_compact_soft_deleted_archive_metadata_missing(archivers, request):
+    # On a damaged repo, compact skips a soft-deleted archive whose metadata object is gone and
+    # goes on compacting (#10435).
+    archiver = request.getfixturevalue(archivers)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "kept_dir/kept_file", contents=b"K" * (1024 * 80))
+    create_regular_file(archiver.input_path, "gone_dir/gone_file", contents=b"G" * (1024 * 80))
+    cmd(archiver, "create", "kept", "input/kept_dir")
+    cmd(archiver, "create", "gone", "input/gone_dir")
+    cmd(archiver, "delete", "gone")  # soft-delete
+
+    repository = open_repository(archiver)
+    with repository:
+        manifest = Manifest.load(repository)
+        kept = Archive(manifest, manifest.archives.get_one(["kept"]).id)
+        victim = next(id for item in kept.iter_items() if "chunks" in item for id, _ in item.chunks)
+        gone_id = manifest.archives.get_one(["gone"], deleted=True).id
+    # damage the repo: a content chunk of the live "kept" archive (so compact sees missing objects and
+    # preserves the soft-deleted archives) and the metadata object of the soft-deleted "gone" archive.
+    cmd(archiver, "debug", "delete-obj", bin_to_hex(victim))
+    cmd(archiver, "debug", "delete-obj", bin_to_hex(gone_id))
+
+    output = cmd(archiver, "compact", "-v", exit_code=EXIT_ERROR)
+    assert "missing objects" in output
+    assert f"Soft-deleted archive archive-does-not-exist {bin_to_hex(gone_id)} cannot be fully preserved" in output
+    assert f"Archive {bin_to_hex(gone_id)} does not exist" in output  # the error shows the id as hex
+    assert "Finished compaction" in output
+
+
 def test_compact_keeps_corrupt_pack(archivers, request):
     # compact keeps a pack "borg check" recorded corrupt unchanged and warns (#10410).
     archiver = request.getfixturevalue(archivers)

src/borg/testsuite/archiver/repo_list_cmd_test.py

@@ -199,6 +199,29 @@ def test_repo_list_deleted(archivers, request, backup_files):
     assert "deleted2" in output
 
 
+def test_repo_list_archive_metadata_missing(archivers, request, backup_files):
+    # an archive whose metadata object is gone is listed with its placeholder values, the listing goes on (#10435).
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    for name in ("normal1", "lost1", "deleted1", "lost2"):
+        cmd(archiver, "create", name, backup_files)
+    ids = dict(line.split() for line in cmd(archiver, "repo-list", "--format={archive} {id}{NL}").splitlines())
+    cmd(archiver, "delete", "deleted1")
+    cmd(archiver, "delete", "lost2")
+    cmd(archiver, "debug", "delete-obj", ids["lost1"])  # a live archive
+    cmd(archiver, "debug", "delete-obj", ids["lost2"])  # a soft-deleted archive
+    output = cmd(archiver, "repo-list")
+    assert "normal1" in output
+    assert "archive-does-not-exist" in output
+    assert "lost1" not in output
+    output = cmd(archiver, "repo-list", "--deleted")
+    assert "deleted1" in output
+    assert "archive-does-not-exist" in output
+    assert "lost2" not in output
+    archives = json.loads(cmd(archiver, "repo-list", "--json", "--format={archive} {tags} {comment}"))["archives"]
+    assert sorted(archive["name"] for archive in archives) == ["archive-does-not-exist", "normal1"]
+
+
 def test_repo_list_from_borg1(archivers, request, monkeypatch):
     archiver = request.getfixturevalue(archivers)
     if archiver.get_kind() in ["remote", "binary"]: