Add read-only upload history and process reports

README.md

@@ -225,6 +225,24 @@ mapillary_tools process MY_CAPTURE_DIR \
     --cutoff_time 120 \
 ```
 
+To receive a machine-readable inventory of recognized inputs and unsupported
+files without changing the image description format, specify an optional
+process report:
+
+```sh
+mapillary_tools process MY_CAPTURE_DIR \
+    --process_report_path /tmp/mapillary_process_report.json
+```
+
+The version 1 report contains `discovered_file_count`, `processed_file_count`,
+`skipped_file_count`, and `skipped_files`. Recognized files that produce a
+description-level processing error are included in `processed_file_count`.
+Unsupported entries include their filename, category, stable reason code, and
+normalized extension. Hidden/system files and common GPS or metadata sidecars
+are ignored during directory discovery, but an explicitly supplied unsupported
+file is always reported. The report is written only to the requested local path
+and contains absolute source filenames, so treat it as local capture metadata.
+
 ## Upload
 
 After processing you should get the [image description file](#image-description). Pass it to the `upload` command to upload them:
@@ -237,6 +255,27 @@ mapillary_tools upload  MY_CAPTURE_DIR \
     --organization_key "my_organization_id"
 ```
 
+## Check Upload History
+
+To check whether processed image sequences or videos were previously uploaded
+from this device, use the same import paths and description file that would be
+passed to `upload`:
+
+```sh
+mapillary_tools check_upload_history MY_CAPTURE_DIR \
+    --desc_path /tmp/mapillary_image_description.json
+```
+
+The command prints a JSON array with one entry per upload candidate. Each entry
+contains the file type, sequence UUID when applicable, sequence checksum,
+member filenames, the subset found in history as
+`already_uploaded_filenames`, and an `already_uploaded` boolean.
+`already_uploaded` is true only when every member filename is represented in
+local history. For images, this includes files that were members of a larger
+previously uploaded sequence. It reads only the local upload history: it does
+not require authentication, make network requests, upload data, or modify
+history.
+
 # Advanced Usage
 
 ## Local Video Processing

mapillary_tools/commands/__init__.py

@@ -6,6 +6,7 @@
 # ruff: noqa: F401
 from . import (
     authenticate,
+    check_upload_history,
     process,
     process_and_upload,
     sample_video,

mapillary_tools/commands/__main__.py

@@ -16,6 +16,7 @@
 from ..utils import configure_logger, get_app_name
 from . import (
     authenticate,
+    check_upload_history,
     process,
     process_and_upload,
     sample_video,
@@ -27,6 +28,7 @@
 
 mapillary_tools_commands = [
     process,
+    check_upload_history,
     upload,
     sample_video,
     video_process,
@@ -62,13 +64,24 @@ def add_general_arguments(parser, command):
             default=False,
             required=False,
         )
-    elif command in ["upload"]:
+    elif command in ["upload", "check_upload_history"]:
         parser.add_argument(
             "import_path",
             help="Paths to your images or videos.",
             nargs="+",
             type=Path,
         )
+        if command == "check_upload_history":
+            parser.add_argument(
+                "--skip_subfolders",
+                help=(
+                    "Skip all subfolders and import only files in the given "
+                    "IMPORT_PATH."
+                ),
+                action="store_true",
+                default=False,
+                required=False,
+            )
     elif command in ["process", "process_and_upload"]:
         parser.add_argument(
             "import_path",

mapillary_tools/commands/check_upload_history.py

@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+import argparse
+import inspect
+import json
+import sys
+
+from ..upload import check_upload_history
+from .process import bold_text
+
+
+class Command:
+    name = "check_upload_history"
+    help = "Check whether processed data exists in local upload history"
+
+    def add_basic_arguments(self, parser: argparse.ArgumentParser):
+        group = parser.add_argument_group(bold_text("UPLOAD HISTORY OPTIONS"))
+        group.add_argument(
+            "--desc_path",
+            help=(
+                "Path to the description file with processed image and video metadata."
+            ),
+            default=None,
+            required=False,
+        )
+
+    def run(self, vars_args: dict):
+        results = check_upload_history(
+            **{
+                key: value
+                for key, value in vars_args.items()
+                if key in inspect.getfullargspec(check_upload_history).args
+            }
+        )
+        json.dump(results, sys.stdout, separators=(",", ":"))
+        sys.stdout.write("\n")

mapillary_tools/commands/process.py

@@ -9,7 +9,7 @@
 import inspect
 from pathlib import Path
 
-from .. import constants, types
+from .. import constants, process_report, types
 from ..process_geotag_properties import (
     DEFAULT_GEOTAG_SOURCE_OPTIONS,
     process_finalize,
@@ -45,6 +45,16 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser):
             default=False,
             required=False,
         )
+        parser.add_argument(
+            "--process_report_path",
+            help=(
+                "Optional path to write a versioned JSON report describing "
+                "recognized process inputs and unsupported files."
+            ),
+            type=Path,
+            default=None,
+            required=False,
+        )
         parser.add_argument(
             "--filetypes",
             "--file_types",
@@ -217,6 +227,14 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser):
         )
 
     def run(self, vars_args: dict):
+        process_report_path = vars_args.get("process_report_path")
+        if process_report_path is not None:
+            report = process_report.build_process_report(
+                import_path=vars_args["import_path"],
+                skip_subfolders=vars_args.get("skip_subfolders", False),
+            )
+            process_report.write_process_report(process_report_path, report)
+
         metadatas = process_geotag_properties(
             **(
                 {

mapillary_tools/history.py

@@ -63,6 +63,77 @@ def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]:
             return None
 
 
+def _normalize_md5sum(value: T.Any) -> str | None:
+    if not isinstance(value, str) or len(value) != 32:
+        return None
+
+    try:
+        _validate_hexdigits(value)
+    except ValueError:
+        return None
+
+    return value.lower()
+
+
+def find_uploaded_image_md5s(md5sums: T.Iterable[str]) -> set[str]:
+    """Find image checksums stored in upload-history descriptions.
+
+    Upload history is sharded by the checksum of a whole sequence, so checking
+    whether images came from a previously uploaded larger sequence requires a
+    bounded scan of the records. Only requested checksums are retained, and the
+    scan stops as soon as all of them have been found.
+    """
+    if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
+        return set()
+
+    wanted = {
+        normalized
+        for md5sum in md5sums
+        if (normalized := _normalize_md5sum(md5sum)) is not None
+    }
+    if not wanted:
+        return set()
+
+    root = Path(constants.MAPILLARY_UPLOAD_HISTORY_PATH)
+    if not root.is_dir():
+        return set()
+
+    found: set[str] = set()
+    try:
+        history_paths = root.glob("*/*.json")
+        for path in history_paths:
+            try:
+                with path.open("r", encoding="utf-8") as fp:
+                    record = json.load(fp)
+            except (OSError, UnicodeError, json.JSONDecodeError) as ex:
+                LOG.warning("Failed to read upload history %s: %s", path, ex)
+                continue
+
+            if not isinstance(record, dict):
+                LOG.warning("Invalid upload history record %s", path)
+                continue
+
+            descs = record.get("descs")
+            if not isinstance(descs, list):
+                continue
+
+            for desc in descs:
+                if not isinstance(desc, dict):
+                    continue
+                if desc.get("filetype") != types.FileType.IMAGE.value:
+                    continue
+                md5sum = _normalize_md5sum(desc.get("md5sum"))
+                if md5sum in wanted:
+                    found.add(md5sum)
+
+            if found == wanted:
+                break
+    except OSError as ex:
+        LOG.warning("Failed to scan upload history %s: %s", root, ex)
+
+    return found
+
+
 def write_history(
     md5sum: str,
     params: JSONDict,