diff --git a/docs/openedx_content/decisions/0025-backup-restore.rst b/docs/openedx_content/decisions/0025-backup-restore.rst new file mode 100644 index 000000000..be3b636f9 --- /dev/null +++ b/docs/openedx_content/decisions/0025-backup-restore.rst @@ -0,0 +1,46 @@ +25. Learning Package Serialization and Validation Approach +========================================================== + +Context +------- + +Content Libraries map 1:1 to LearningPackages and these need to be imported and exported as file archives. Initial support for this was released in Ulmo, but we wanted to revisit it to make it more robust during the Verawood timeline. This is part of that effort. + +* Flexibility of Structure +* Standardization of validation (JSON Schema) +* Justify ZIP +* Justify TOML +* Max 100,000 items. +* Use of fsspec as abstraction + +Phases + +Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage + + +Decision +-------- + +Some key points: + +1. We intentionally separate input and output formats because the output format + will change over time, but the various input formats must continue to be + supported. We don't inherit from one from the other because we don't *want* + those changes to be automatically propogated--that breaks compatibility. +2. We assemble into giant JSON in order to simplify validation and allow for + more flexibility in structural representation. There's the archive layer and + then the logical layer and then serialization into the database. + + +Archive -> Model (validation) + Resources -> Database + + + +Consequences +------------ + + + +Rejected alternatives +--------------------- + diff --git a/mypy.ini b/mypy.ini index b383a8816..f3994e68d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,5 +12,8 @@ files = [mypy-organizations.*] follow_untyped_imports = True +[mypy-fsspec.*] +follow_untyped_imports = True + [mypy.plugins.django-stubs] django_settings_module = "projects.dev" diff --git a/requirements/base.in b/requirements/base.in index 626a551be..a384a92af 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -17,3 +17,7 @@ rules<4.0 # Django extension for rules-based authorization check tomlkit # Parses and writes TOML configuration files edx-organizations # Implemented the "Organization" model that CatalogCourse/CourseRun are keyed to + +fsspec # Used by openedx_content's backup_restore to abstract zip access + +pydantic[email] # Used by openedx_content's backup_restore for input validation diff --git a/requirements/base.txt b/requirements/base.txt index 43659d809..1cd2dee59 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,6 +6,8 @@ # amqp==5.3.1 # via kombu +annotated-types==0.8.0 + # via pydantic asgiref==3.12.1 # via django attrs==26.1.0 @@ -24,7 +26,7 @@ cffi==2.1.1 # pynacl charset-normalizer==3.5.1 # via requests -click==8.4.2 +click==8.5.0 # via # celery # click-didyoumean @@ -37,7 +39,7 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -cryptography==50.0.0 +cryptography==50.0.1 # via pyjwt django==5.2.17 # via @@ -70,7 +72,9 @@ djangorestframework==3.18.0 # edx-drf-extensions # edx-organizations dnspython==2.8.0 - # via pymongo + # via + # email-validator + # pymongo drf-jwt==1.19.2 # via edx-drf-extensions edx-ccx-keys==2.0.2 @@ -91,10 +95,16 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/base.in +email-validator==2.3.0 + # via pydantic fastavro==1.12.2 # via openedx-events +fsspec==2026.7.0 + # via -r requirements/base.in idna==3.19 - # via requests + # via + # email-validator + # requests kombu==5.6.2 # via celery openedx-events==11.2.0 @@ -109,6 +119,10 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi +pydantic[email]==2.13.5 + # via -r requirements/base.in +pydantic-core==2.46.5 + # via pydantic pyjwt[crypto]==2.13.0 # via # drf-jwt @@ -138,7 +152,13 @@ stevedore==5.9.1 tomlkit==0.15.1 # via -r requirements/base.in typing-extensions==4.16.0 - # via edx-opaque-keys + # via + # edx-opaque-keys + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via pydantic tzdata==2026.3 # via kombu tzlocal==5.4.4 @@ -150,7 +170,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via prompt-toolkit # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/ci.txt b/requirements/ci.txt index 8ba2a927b..044e40258 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -19,7 +19,7 @@ packaging==26.3 # via # pyproject-api # tox -platformdirs==4.11.3 +platformdirs==4.11.5 # via # tox # virtualenv @@ -27,15 +27,15 @@ pluggy==1.6.0 # via tox pyproject-api==1.11.0 # via tox -python-discovery==1.5.2 +python-discovery==1.6.0 # via # tox # virtualenv tomli-w==1.2.0 # via tox -tox==4.60.0 +tox==4.60.1 # via -r requirements/ci.in typing-extensions==4.16.0 # via tox -virtualenv==21.7.4 +virtualenv==21.7.7 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index c47d8e064..28854da8c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/quality.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/quality.txt + # pydantic asgiref==3.12.1 # via # -r requirements/quality.txt @@ -29,7 +33,7 @@ billiard==4.2.4 # via # -r requirements/quality.txt # celery -build==1.5.0 +build==1.6.0 # via # -r requirements/pip-tools.txt # pip-tools @@ -54,7 +58,7 @@ charset-normalizer==3.5.1 # via # -r requirements/quality.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt @@ -96,7 +100,7 @@ coverage[toml]==7.15.4 # via # -r requirements/quality.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/quality.txt # pyjwt @@ -166,11 +170,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/quality.txt dnspython==2.8.0 # via # -r requirements/quality.txt + # email-validator # pymongo docutils==0.23 # via @@ -206,6 +211,10 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/quality.txt +email-validator==2.3.0 + # via + # -r requirements/quality.txt + # pydantic fastavro==1.12.2 # via # -r requirements/quality.txt @@ -218,7 +227,9 @@ filelock==3.32.4 # virtualenv freezegun==1.5.5 # via -r requirements/quality.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/quality.txt +grimp==3.16 # via # -r requirements/quality.txt # import-linter @@ -229,8 +240,9 @@ id==1.6.1 idna==3.19 # via # -r requirements/quality.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/quality.txt iniconfig==2.3.0 # via @@ -341,7 +353,7 @@ pillow==12.3.0 # edx-organizations pip-tools==7.6.1 # via -r requirements/pip-tools.txt -platformdirs==4.11.3 +platformdirs==4.11.5 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -372,6 +384,12 @@ pycparser==3.0 # via # -r requirements/quality.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/quality.txt +pydantic-core==2.46.5 + # via + # -r requirements/quality.txt + # pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.21.0 @@ -437,7 +455,7 @@ python-dateutil==2.9.0.post0 # -r requirements/quality.txt # celery # freezegun -python-discovery==1.5.2 +python-discovery==1.6.0 # via # -r requirements/ci.txt # tox @@ -451,7 +469,7 @@ pyyaml==6.0.3 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==45.0 +readme-renderer==46.0 # via # -r requirements/quality.txt # twine @@ -518,7 +536,7 @@ tomlkit==0.15.1 # -r requirements/quality.txt # edx-lint # pylint -tox==4.60.0 +tox==4.60.1 # via -r requirements/ci.txt twine==7.0.0 # via -r requirements/quality.txt @@ -537,7 +555,14 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core # tox + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/quality.txt + # pydantic tzdata==2026.3 # via # -r requirements/quality.txt @@ -558,11 +583,11 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==21.7.4 +virtualenv==21.7.7 # via # -r requirements/ci.txt # tox -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/quality.txt # prompt-toolkit diff --git a/requirements/doc.txt b/requirements/doc.txt index 99e4bd15c..f3b2f4a67 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,6 +12,10 @@ amqp==5.3.1 # via # -r requirements/test.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/test.txt + # pydantic asgiref==3.12.1 # via # -r requirements/test.txt @@ -49,7 +53,7 @@ charset-normalizer==3.5.1 # via # -r requirements/test.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/test.txt # celery @@ -77,7 +81,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/test.txt # pyjwt @@ -134,11 +138,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.txt dnspython==2.8.0 # via # -r requirements/test.txt + # email-validator # pymongo doc8==2.0.0 # via -r requirements/doc.in @@ -175,23 +180,30 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/test.txt +email-validator==2.3.0 + # via + # -r requirements/test.txt + # pydantic fastavro==1.12.2 # via # -r requirements/test.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/test.txt +grimp==3.16 # via # -r requirements/test.txt # import-linter idna==3.19 # via # -r requirements/test.txt + # email-validator # requests -imagesize==2.0.0 +imagesize==2.0.1 # via sphinx -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.txt iniconfig==2.3.0 # via @@ -270,6 +282,12 @@ pycparser==3.0 # via # -r requirements/test.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/test.txt +pydantic-core==2.46.5 + # via + # -r requirements/test.txt + # pydantic pydata-sphinx-theme==0.20.0 # via sphinx-book-theme pygments==2.21.0 @@ -317,7 +335,7 @@ pyyaml==6.0.3 # via # -r requirements/test.txt # code-annotations -readme-renderer==45.0 +readme-renderer==46.0 # via -r requirements/doc.in requests==2.34.2 # via @@ -403,6 +421,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/test.txt + # pydantic tzdata==2026.3 # via # -r requirements/test.txt @@ -421,7 +446,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 9e327bfba..008bd5b92 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,9 +4,9 @@ # # make upgrade # -build==1.5.0 +build==1.6.0 # via pip-tools -click==8.4.2 +click==8.5.0 # via pip-tools packaging==26.3 # via diff --git a/requirements/quality.txt b/requirements/quality.txt index ad1da6fd9..7e2f9d426 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/test.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/test.txt + # pydantic asgiref==3.12.1 # via # -r requirements/test.txt @@ -43,7 +47,7 @@ charset-normalizer==3.5.1 # via # -r requirements/test.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/test.txt # celery @@ -77,7 +81,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/test.txt # pyjwt @@ -136,11 +140,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.txt dnspython==2.8.0 # via # -r requirements/test.txt + # email-validator # pymongo docutils==0.23 # via readme-renderer @@ -172,13 +177,19 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/test.txt +email-validator==2.3.0 + # via + # -r requirements/test.txt + # pydantic fastavro==1.12.2 # via # -r requirements/test.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/test.txt +grimp==3.16 # via # -r requirements/test.txt # import-linter @@ -187,8 +198,9 @@ id==1.6.1 idna==3.19 # via # -r requirements/test.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.txt iniconfig==2.3.0 # via @@ -270,7 +282,7 @@ pillow==12.3.0 # via # -r requirements/test.txt # edx-organizations -platformdirs==4.11.3 +platformdirs==4.11.5 # via pylint pluggy==1.6.0 # via @@ -291,6 +303,12 @@ pycparser==3.0 # via # -r requirements/test.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/test.txt +pydantic-core==2.46.5 + # via + # -r requirements/test.txt + # pydantic pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.21.0 @@ -348,7 +366,7 @@ pyyaml==6.0.3 # via # -r requirements/test.txt # code-annotations -readme-renderer==45.0 +readme-renderer==46.0 # via twine requests==2.34.2 # via @@ -417,6 +435,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/test.txt + # pydantic tzdata==2026.3 # via # -r requirements/test.txt @@ -437,7 +462,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 6a1fe812f..33ea67cc8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/base.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/base.txt + # pydantic asgiref==3.12.1 # via # -r requirements/base.txt @@ -37,7 +41,7 @@ charset-normalizer==3.5.1 # via # -r requirements/base.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/base.txt # celery @@ -65,7 +69,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.in # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/base.txt # pyjwt @@ -118,11 +122,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.in dnspython==2.8.0 # via # -r requirements/base.txt + # email-validator # pymongo drf-jwt==1.19.2 # via @@ -150,19 +155,26 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/base.txt +email-validator==2.3.0 + # via + # -r requirements/base.txt + # pydantic fastavro==1.12.2 # via # -r requirements/base.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.in -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/base.txt +grimp==3.16 # via import-linter idna==3.19 # via # -r requirements/base.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.in iniconfig==2.3.0 # via pytest @@ -217,6 +229,12 @@ pycparser==3.0 # via # -r requirements/base.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/base.txt +pydantic-core==2.46.5 + # via + # -r requirements/base.txt + # pydantic pygments==2.21.0 # via # pytest @@ -297,6 +315,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/base.txt + # pydantic tzdata==2026.3 # via # -r requirements/base.txt @@ -315,7 +340,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/base.txt # prompt-toolkit diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 8f880aa7b..dd99352e5 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -1,42 +1,132 @@ """ Backup Restore API + +This module is responsible for creating a backup archive of a Learning Package, +as well as creating a new Learning Package based on a backup archive file. """ -import zipfile +from dataclasses import asdict +from datetime import datetime, timezone +from io import StringIO from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user +from django.db.transaction import atomic -from ..publishing.api import get_learning_package_by_ref -from .zipper import LearningPackageUnzipper, LearningPackageZipper +from ..publishing import api as publishing_api +from . import archive, loading, payload, validation +from .errors import BackupRestoreError, RestoreFailedError +from .results import RestoreResult, generate_staged_package_ref +from .zipper import LearningPackageZipper -# The public API that will be re-exported by openedx_content.api -# is listed in the __all__ entries below. Internal helper functions that are -# private to this module should start with an underscore. If a function does not -# start with an underscore AND it is not in __all__, that function is considered -# to be callable only by other applets in the openedx_content package. __all__ = [ "create_zip_file", + "load_learning_package_from_path", "load_learning_package", ] +def load_learning_package_from_path( + path_str: str, + user: UserType, + package_ref: str | None = None, +) -> RestoreResult: + """ + Loads a learning package from a file system at the given path. + + The ``path_str`` will usually point to a Zip file archive that holds the + backup of the Learning Package data. For testing and debugging purposes, you + can also specify ``path_str`` to be the root directory of an unzipped + version of the archive data. + + The overall pipeline looks like this: + + archive.py: Archive location (Path) → FileSystem (fsspec) + payload.py: FileSystem (fsspec) → UnvalidatedLearningPackageInput + validation.py: UnvalidatedLearningPackageInput → ValidatedLearningPackageInput + loading.py: ValidatedLearningPackageInput → LearningPackage + + If ``package_ref`` is not supplied, we generate a staged one namespaced to + ``user``. We can't just use the ref from the archive, because the archive + can claim any ref it likes and the user may not be allowed to create it. + + Errors that can be raised: + + * ``ArchiveNotReadableError`` if we can't open ``path_str`` at all. + * ``RestoreFailedError`` if the archive's contents don't validate. Nothing + is written to the database in that case. + + Both descend from ``BackupRestoreError``. + """ + fs = archive.read_fs_for_path(path_str) + unvalidated_input = payload.PayloadExtractor(fs).extract() + validated_input = validation.validate(unvalidated_input) + + # Bail out before touching the database. We deliberately don't do a partial + # restore: a half-loaded Learning Package is harder to reason about than no + # Learning Package at all. + if validated_input.errors: + raise RestoreFailedError(validated_input.errors, validated_input.fs.path) + + loader = loading.Loader(validated_input) + archive_lp_input = loader.data.learning_package # LearningPackageInputData + if package_ref is None: + package_ref = generate_staged_package_ref(archive_lp_input.key, user) + + now = datetime.now(tz=timezone.utc) + with atomic(savepoint=False): + learning_package = publishing_api.create_learning_package( + package_ref, + archive_lp_input.title, + description=archive_lp_input.description or "", + created=archive_lp_input.created or now, + ) + load_target = loading.Loader.Target(learning_package, user, now) + result = loader.load_into(load_target) + + return result + + +def load_learning_package( + path_str: str, + user: UserType, + package_ref: str | None = None, +) -> dict: + """ + ``create_learning_package``, in the dict shape that 1.0 clients expect. + + Returns a dict with the status of the operation and any errors encountered + during that process, rather than raising. + + TODO: This exists so that callers written against the pre-pydantic restore + keep working. New callers should use ``load_learning_package`` and catch + ``BackupRestoreError``, so that this can eventually go away. + """ + try: + result = load_learning_package_from_path(path_str, user, package_ref) + except RestoreFailedError as err: + return asdict( + RestoreResult(status="error", log_file_error=StringIO(err.as_text())) + ) + except BackupRestoreError as err: + return asdict( + RestoreResult(status="error", log_file_error=StringIO(f"{err}\n")) + ) + + return asdict(result) + + def create_zip_file( - package_ref: str, path: str, user: UserType | None = None, origin_server: str | None = None + lp_key: str, + path: str, + user: UserType | None = None, + origin_server: str | None = None, ) -> None: """ Creates a dump zip file for the given learning package key at the given path. The zip file contains a TOML representation of the learning package and its contents. + This is used by lp_dump. + Can throw a NotFoundError at get_learning_package_by_ref """ - learning_package = get_learning_package_by_ref(package_ref) + learning_package = publishing_api.get_learning_package_by_ref(lp_key) LearningPackageZipper(learning_package, user, origin_server).create_zip(path) - - -def load_learning_package(path: str, package_ref: str | None = None, user: UserType | None = None) -> dict: - """ - Loads a learning package from a zip file at the given path. - Restores the learning package and its contents to the database. - Returns a dictionary with the status of the operation and any errors encountered. - """ - with zipfile.ZipFile(path, "r") as zipf: - return LearningPackageUnzipper(zipf, package_ref, user).load() diff --git a/src/openedx_content/applets/backup_restore/archive.py b/src/openedx_content/applets/backup_restore/archive.py new file mode 100644 index 000000000..1538965b9 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/archive.py @@ -0,0 +1,37 @@ +""" +This module exists to abstract away the container archive format. To being with, +we are supporting Zip files and simple directories (useful for testing). +""" +from pathlib import Path + +from fsspec import AbstractFileSystem +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem + +from .errors import ArchiveNotReadableError + + +def read_fs_for_path(path_str: str) -> AbstractFileSystem: + """ + If the path_str passed in is a directory, we treat that as the root of the + archive to be restored. Otherwise, we assume you're passing a Zip file. + + For future consideration: Using LibArchiveFileSystem would allow us to + support tar.gz, zip, 7z, and a bunch of other archiving formats in read-only + mode. I'm not doing it now because I'm not clear on whether the reliance on + libarchive makes things problematic, I don't understand the performance + implications, and I don't want to open the door on "supported archive + formats" to include everything under the sun. But it's an intriguing option + to consider. + """ + path = Path(path_str) + if path.is_dir(): + # read-only mode is not available for DirFileSystem + return DirFileSystem(path) + elif path.is_file() and path.suffix.lower() == ".zip": + # read-only is the default for ZipFilesystem, but make it explicit + return ZipFileSystem(path, mode="r") + + raise ArchiveNotReadableError( + "Expected a directory or a .zip file", path=path_str + ) diff --git a/src/openedx_content/applets/backup_restore/errors.py b/src/openedx_content/applets/backup_restore/errors.py new file mode 100644 index 000000000..d7fb1baea --- /dev/null +++ b/src/openedx_content/applets/backup_restore/errors.py @@ -0,0 +1,205 @@ +""" +Errors raised while backing up or restoring a Learning Package. + +Everything in this applet raises something that descends from +:class:`BackupRestoreError`, so callers that don't care about the specifics can +catch a single type. The subclasses exist to give useful debug output and to let +tests assert on a specific failure rather than on message text. + +The hierarchy is grouped by the pipeline stage that detects the problem: + +* :class:`ArchiveNotReadableError` -- ``archive.py``, we can't open the thing at all. +* :class:`ExtractionError` -- ``payload.py``, the archive's files are malformed in a + way that stops us assembling the input document (unparseable TOML, duplicate + entity definitions, missing mandatory files). +* :class:`SchemaError` -- ``validation.py``, a field failed pydantic validation. +* :class:`ConsistencyError` -- ``validation.py``, a cross-reference check that + pydantic can't express (e.g. a container pointing at a child that isn't in the + archive). + +:class:`RestoreFailedError` is the aggregate that the public API raises. It holds +all of the individual errors found during a single restore attempt. +""" +from __future__ import annotations + + +class BackupRestoreError(Exception): + """ + Base class for every error this applet raises. + + Args: + message: Human-readable description of what went wrong. + path: Archive-relative path of the file the problem was found in, e.g. + ``"entities/unit1-b7eafb.toml"``. ``None`` when the error isn't + attributable to a particular file. + """ + + def __init__(self, message, path=None): + super().__init__(message) + self.message = message + self.path = path + + def __str__(self): + # Not every error is attributable to a file -- a duplicate key, for + # instance, is reported against the whole section rather than one path. + # Printing "None: ..." in a log file helps nobody. + if self.path: + return f"{self.path}: {self.message}" + return self.message + + +class ArchiveNotReadableError(BackupRestoreError): + """We could not open the archive at all (not a directory, not a zip file).""" + + +# --- Extraction Errors (payload.py) --- + + +class ExtractionError(BackupRestoreError): + """ + Any error during the extraction process. + + At the moment, any error is fatal. The point of the different errors is to + provide useful debug logging and to let us write tests that look for + specific errors. + """ + + +class InvalidTOMLError(ExtractionError): + def __init__(self, file_description, details, path): + message = f"Cannot decode TOML for {file_description}: {details}" + super().__init__(message, path=path) + + +class TableNotFoundError(ExtractionError): + def __init__(self, file_description, table, path): + self.table = table + message = f"Table [{table}] not found in {file_description}." + super().__init__(message, path=path) + + +class FieldsNotInTable(ExtractionError): + def __init__(self, file_description, fields, path): + self.fields = sorted(fields) + message = f"{file_description} has fields not in a table: {', '.join(fields)}" + super().__init__(message, path=path) + + +class MissingFileError(ExtractionError): + """ + A file we require is not in the archive. + + Note: this used to be called ``FileNotFoundError``, which shadowed the + builtin of the same name and made ``except FileNotFoundError`` ambiguous for + our callers. + """ + + def __init__(self, file_description, path): + message = f"{file_description} file not found at expected path" + super().__init__(message, path=path) + + +class UnsupportedFormatError(ExtractionError): + """The archive declares a ``format_version`` we don't know how to read.""" + + +# --- Validation Errors (validation.py) --- + + +class SchemaError(BackupRestoreError): + """ + A field failed pydantic validation. + + One of these is created for each entry in a ``pydantic.ValidationError``, so + that we can attribute the failure to the archive file it came from instead + of reporting a JSON pointer into a document the user never sees. + + Args: + message: pydantic's error message for this entry. + path: Archive-relative path of the source file, if we can work it out. + location: The part of pydantic's ``loc`` tuple that is meaningful + *within* that file, e.g. ``("versions", 0, "title")``. + """ + + def __init__(self, message, path=None, location=()): + self.location = tuple(location) + super().__init__(message, path=path) + + def __str__(self): + parts = [] + if self.path: + parts.append(str(self.path)) + if self.location: + parts.append(".".join(str(part) for part in self.location)) + parts.append(self.message) + return ": ".join(parts) + + +class ConsistencyError(BackupRestoreError): + """ + A cross-reference in the archive doesn't hold up. + + These are the checks that pydantic can't express, because they involve more + than one part of the document at once. + """ + + +class UnresolvedChildError(ConsistencyError): + """A container version lists a child that isn't defined in the archive.""" + + +class MissingVersionError(ConsistencyError): + """An entity's draft or published pointer names a version that isn't in the archive.""" + + +class DuplicateVersionError(ConsistencyError): + """An entity declares the same ``version_num`` more than once.""" + + +class MalformedRefError(ConsistencyError): + """An entity ref isn't in a shape we know how to load.""" + + +class UnknownContainerTypeError(ConsistencyError): + """The archive declares a container type this version of the code can't load.""" + + +# --- Aggregate --- + + +class RestoreFailedError(BackupRestoreError): + """ + The restore could not be completed. Holds every error we found. + + We deliberately gather as many problems as we can before raising, so that + someone fixing up an archive by hand doesn't have to discover their mistakes + one run at a time. + """ + + def __init__( + self, + errors: list[BackupRestoreError], + archive_root: str | None = None, + ): + self.errors = list(errors) + # The folder inside the archive we treated as the root, when the archive + # wrapped its contents in one. Every path below is relative to it, so + # saying so once up front saves a lot of confusion. + self.archive_root = archive_root + super().__init__(f"Restore failed with {len(self.errors)} error(s).") + + def __str__(self): + return self.as_text() + + def as_text(self) -> str: + """ + Render every error as a block of text suitable for a log file. + + The format matches what the pre-pydantic implementation wrote out, so + that existing consumers of the restore log keep working. + """ + lines = ["Errors encountered during restore:"] + if self.archive_root: + lines.append(f"Archive root: {self.archive_root}/") + lines.extend(str(err) for err in self.errors) + return "\n".join(lines) + "\n" diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py new file mode 100644 index 000000000..ccc1153e8 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -0,0 +1,369 @@ +""" +Logic for taking the logical schema model for a Learning Package and loading it +into the database. +""" +import mimetypes +import os.path +from dataclasses import dataclass +from datetime import datetime, timedelta +from functools import cache, partial + +from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user +from django.db.transaction import atomic + +from ..collections import api as collections_api +from ..components import api as components_api +from ..containers import api as containers_api +from ..media import api as media_api +from ..media.models import Media +from ..publishing import api as publishing_api +from ..publishing.models import LearningPackage +from ..sections.models import Section +from ..subsections.models import Subsection +from ..units.models import Unit +from .errors import UnknownContainerTypeError +from .results import BackupMetadata, RestoreLearningPackageData, RestoreResult, unpack_package_ref +from .schema import EntityInputData, SectionInputData, SubsectionInputData, UnitInputData +from .validation import ValidatedLearningPackageInput + + +class Loader: + """ + Loads the validated input into a Learning Package in the database. + + This class does not understand the specifics of the archive file format. It + only needs the ValidatedLearningPackageInput. + """ + + @dataclass(frozen=True) + class Target: + learning_package: LearningPackage + user: UserType + loaded_at: datetime + + def __init__(self, validated_input: ValidatedLearningPackageInput): + if validated_input.data is None: + raise ValueError( + "Cannot load a LearningPackage from input that failed validation." + ) + self.validated_input = validated_input + self.data = validated_input.data + self.component_inputs: dict[str, EntityInputData] = {} + self.section_inputs: dict[str, EntityInputData] = {} + self.subsection_inputs: dict[str, EntityInputData] = {} + self.unit_inputs: dict[str, EntityInputData] = {} + + # Sorted so that creation order stays deterministic regardless of the + # order the archive's files happened to be read in. + entity_inputs = sorted(self.data.entities, key=lambda entity: entity.key) + self.entity_inputs_by_ref = {entity.key: entity for entity in entity_inputs} + + # Split our entities into separate dicts for convenience. + for entity_input in entity_inputs: + entity_ref = entity_input.key + match entity_input.container: + case SectionInputData(): + self.section_inputs[entity_ref] = entity_input + case SubsectionInputData(): + self.subsection_inputs[entity_ref] = entity_input + case UnitInputData(): + self.unit_inputs[entity_ref] = entity_input + case None: + # For the moment, if it's not a Container, it's a Component + self.component_inputs[entity_ref] = entity_input + case _: + # validation.py should have rejected this already. + raise UnknownContainerTypeError( + f'Cannot load entity "{entity_ref}": unrecognized container type', + ) + + def load_into(self, target: Target) -> RestoreResult: + """ + This method intentionally takes a target (LearningPackage, User, + Datetime) instead of putting that information into Loader object state. + My hope is that this pattern will make it easier to adapt into handling + incremental imports where we have to test the same input being imported + into multiple Learning Packages with existing state. + """ + with atomic(savepoint=False): + bulk_change_context_for_time = partial( + publishing_api.bulk_draft_changes_for, + target.learning_package.id, + changed_by=target.user.id, + ) + + # DraftChangeLog 1: Add all the PublishableEntities and their versions, + # and set their versions to prepare for for publishing. + with bulk_change_context_for_time(changed_at=target.loaded_at): + loaded_components = self.load_components_into(target) + loaded_entities = self.load_containers_into(target, loaded_components) + self.set_draft_versions(target, for_publishing=True) + + publishing_api.publish_all_drafts( + target.learning_package.id, + published_at=target.loaded_at, + published_by=target.user.id, + message="Restore from backup.", + ) + + # DraftChangeLog 2: Set all PublishableEntities to their proper draft. + # At this point, all versions have been loaded, and the correct versions + # have been published, but the current draft version might be wrong. + # + # The history display will want draft changes to be slightly after the + # published log entry. + changed_at = target.loaded_at + timedelta(seconds=1) + with bulk_change_context_for_time(changed_at=changed_at): + self.set_draft_versions(target, for_publishing=False) + + # Collections are added at the end, in case publishing of contents would + # cause more thrashing w.r.t. search indexing. + self.load_collections_into(target, loaded_entities) + + return self.build_restore_result(target) + + def build_restore_result(self, target: Target) -> RestoreResult: + """ + Summarize what we just loaded. + + TODO: The shape of RestoreResult is inherited from what we already send + to the frontend. We should return something more structured here and let + api.py handle the translation into what the REST API expects. + """ + validated_data = self.data + + org_code, package_code = unpack_package_ref(validated_data.learning_package.key) + + loaded_entities = publishing_api.get_publishable_entities(target.learning_package.id) + + result = RestoreResult( + status="success", + log_file_error=None, + lp_restored_data=RestoreLearningPackageData( + id=target.learning_package.id, + package_ref=target.learning_package.package_ref, + archive_package_ref=validated_data.learning_package.key, + archive_org_code=org_code, + archive_package_code=package_code, + title=target.learning_package.title, + num_containers=loaded_entities.filter(container__isnull=False).count(), + num_sections=loaded_entities.filter(container__section__isnull=False).count(), + num_subsections=loaded_entities.filter(container__subsection__isnull=False).count(), + num_units=loaded_entities.filter(container__unit__isnull=False).count(), + num_components=loaded_entities.filter(component__isnull=False).count(), + num_collections=collections_api.get_collections(target.learning_package.id).count(), + ), + backup_metadata=BackupMetadata( + format_version=validated_data.meta.format_version, + created_by=validated_data.meta.created_by, + created_by_email=validated_data.meta.created_by_email, + created_at=validated_data.meta.created_at, + original_server=validated_data.meta.origin_server, + ), + ) + return result + + def load_components_into(self, target: Target): + """ + Create every Component and its versions, and return them by ref. + + The returned mapping is what containers use to resolve their + children, so this has to run before any container is loaded. + """ + + @cache # inner fn, so won't persist across calls to load_components_into + def _get_component_type(namespace: str, name: str): + return components_api.get_or_create_component_type(namespace, name) + + @cache # inner fn, so won't persist across calls to load_components_into + def _get_media_type(mime_type: str): + return media_api.get_or_create_media_type(mime_type) + + mapping = {} + for entity_ref, entity_input in self.component_inputs.items(): + namespace, name, component_code = entity_ref.split(":") + component_type = _get_component_type(namespace, name) + component = components_api.create_component( + target.learning_package.id, + component_type=component_type, + component_code=component_code, + created=target.loaded_at, + created_by=target.user.id, + ) + # TODO: Validate missing children + sorted_version_inputs = sorted( + entity_input.versions, key=lambda v: v.version_num + ) + for version_input in sorted_version_inputs: + media_to_replace: dict[str, Media.ID | Media | bytes | None] = {} + version_media = ( + version_input.component.media if version_input.component else {} + ) + for path, text_val in version_media.items(): + filename = os.path.basename(path) + if filename == "block.xml": + media_type = _get_media_type( + f"application/vnd.openedx.xblock.v1.{component_type.name}+xml" + ) + else: + media_type_str, _encoding = mimetypes.guess_type(filename) + media_type_str = media_type_str or "application/octet-stream" + media_type = _get_media_type(media_type_str) + + if path.startswith('static/'): + # TODO: data-url support + # This is where we could add base64 encoded versions + # when we introduce data-url support. But for now, just + # just assume these are file paths, e.g. path/to/file + filepath = text_val + new_media = media_api.get_or_create_file_media( + target.learning_package.id, + media_type.id, + data=self.validated_input.fs.read_bytes(filepath), + created=target.loaded_at, + ) + else: + new_media = media_api.get_or_create_text_media( + target.learning_package.id, + media_type.id, + text=text_val, + created=target.loaded_at, + ) + + media_to_replace[path] = new_media.id + + # TODO: Modify create_next_component_version to take a Component + # as an option, to save the needless fetches. + components_api.create_next_component_version( + component.id, + title=version_input.title, + media_to_replace=media_to_replace, + created=target.loaded_at, + created_by=target.user.id, + force_version_num=version_input.version_num, + ) + mapping[entity_ref] = component + + return mapping + + def load_containers_into(self, target: Target, component_mapping: dict): + """ + Create every Container and its versions, and return them by ref. + + Containers are built bottom-up (Units, then Subsections, then + Sections), because each level references the one below it. + """ + + # Ordering matters, since we want to build the references bottom-up. + container_types_to_inputs = { + Unit: self.unit_inputs, + Subsection: self.subsection_inputs, + Section: self.section_inputs, + } + mapping = component_mapping.copy() + for container_type, container_inputs in container_types_to_inputs.items(): + for entity_ref, entity_input in container_inputs.items(): + container = containers_api.create_container( + target.learning_package.id, + entity_ref, + created=target.loaded_at, + created_by=target.user.id, + container_cls=container_type, + ) + + # TODO: Validate missing children + sorted_version_inputs = sorted( + entity_input.versions, key=lambda v: v.version_num + ) + for version_input in sorted_version_inputs: + containers_api.create_next_container_version( + container, + title=version_input.title, + entities=[ + mapping[child_ref] + for child_ref in ( + version_input.container.children + if version_input.container + else [] + ) + ], + created=target.loaded_at, + created_by=target.user.id, + force_version_num=version_input.version_num, + ) + + mapping[entity_ref] = container + + return mapping + + def load_collections_into(self, target: Target, loaded_entities): + """ + Create Collections and add their members. + + Members that aren't in the archive are skipped rather than treated + as an error -- a Collection with a missing entry is still usable. + """ + for collection_input in self.data.collections: + collections_api.create_collection( + target.learning_package.id, + collection_code=collection_input.key, + title=collection_input.title, + created_by=target.user.id, + description=collection_input.description or "", + ) + loaded_entity_refs = [ + ref for ref in collection_input.entities if ref in loaded_entities + ] + entities = publishing_api.get_publishable_entities( + target.learning_package.id + ).filter(entity_ref__in=loaded_entity_refs) + + collections_api.add_to_collection( + target.learning_package.id, + collection_code=collection_input.key, + entities_qset=entities, + ) + + def set_draft_versions(self, target: Target, for_publishing: bool): + """ + Point every entity's draft at the version the archive asks for. + + This runs twice. The first pass (``for_publishing=True``) sets each + draft to the version that should end up published, so that + ``publish_all_drafts`` publishes the right thing. The second pass + sets the drafts to their real values. + """ + saved_entities = publishing_api.get_publishable_entities( + target.learning_package.id + ) + for saved_entity in saved_entities: + saved_draft_version = publishing_api.get_draft_version(saved_entity) + input_entity = self.entity_inputs_by_ref[saved_entity.entity_ref] + + if for_publishing: + input_version_num = input_entity.published.version_num + else: + input_version_num = input_entity.draft.version_num + + # The version we want to set is already the current draft, which + # means there's nothing to do. + if ( + saved_draft_version + and saved_draft_version.version_num == input_version_num + ): + continue + + if input_version_num is None: + version_id_to_set = None + else: + version_model_to_publish = saved_entity.versions.get( + version_num=input_version_num + ) + version_id_to_set = version_model_to_publish.id + + publishing_api.set_draft_version( + saved_entity.id, + version_id_to_set, + set_at=target.loaded_at, + set_by=target.user.id, + ) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py new file mode 100644 index 000000000..eb2cf8923 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -0,0 +1,539 @@ +""" +This module works with the actual files in our backup archive. It is agnostic to +the archive container format that the files are bundled in, e.g. a local file +system directory, a zip file archive, or something more exotic down the line. + +The error checking here is for the file format itself, i.e. extracting values +from the TOML files and static assets and assembling them for validation. +""" + +from __future__ import annotations + +import os.path # fsspec doesn't work well with Path objects. +import tomllib + +import attrs +from fsspec import AbstractFileSystem +from fsspec.implementations.dirfs import DirFileSystem + +from .errors import ( + ExtractionError, + FieldsNotInTable, + InvalidTOMLError, + MissingFileError, + TableNotFoundError, + UnsupportedFormatError, +) + + +@attrs.define(frozen=True) +class UnvalidatedLearningPackageInput: + """ + Everything we could pull out of an archive, before validation. + + ``raw_data`` is the assembled document we hand to pydantic validation (into + the ``CompletePackageInputData`` model). + ``errors`` holds anything that stopped us assembling part of it. + """ + raw_data: dict + errors: list[ExtractionError] + + # Always the re-rooted filesystem, never the one we were handed. ``fs.path`` + # is the folder inside the archive that we treated as the root, or "" if the + # contents were already at the top level -- so anyone who wants to tell a + # human what we decided can read it from here. Every path in ``raw_data`` + # and ``errors`` is relative to this root. + fs: DirFileSystem + + +class PayloadExtractor: + """ + Extracts files from a file system and generates unvalidated input. + + We scan through the archive and compile a Python dictionary that can be + validated against CompletePackageInputData. This mostly involves reading a + bunch of TOML-serialized files and copying their contents, with some minor + data transformations where idiomatic TOML doesn't really give the structure + we want in our final model. + + The purpose of this abstraction is to later allow different ways to assemble + the JSON that we want to do our validation on. By default, this is a bunch + of TOML files, but folks who have specialized authoring needs may prefer a + different set of conventions. For instance, the MIT Disciplinary Experts in + Learning Technology and Applications team prefers to author in a way that + encodes large parts of the hierarchy (Section -> Subsection -> Unit) in a + single file, with pointers to certain Components in different files. We are + not making this fully pluggable yet, but by keeping the payload extraction + separated from the archive container type (archive.py) on one side, and the + schema validation rules (validation.py) on the other side, we have the + flexbility to make the payload handling pluggable later on. + + Errors can happen at this layer, but they are errors related to the + consistency of the archive payload format itself. So errors that need to be + checked here are things like: + + * Missing critical files, like package.toml + * Duplicated entity files, as this is not possible to represent in the + CompletePackageInputData schema. + + Things like missing fields and incorrect field values will be handled at the + validation step which happens after this. In other words, the only things + that are errors here are the things that prevent us from creating a + UnvalidatedLearningPackageInput at all. + """ + ROOT_PACKAGE_PATH = "package.toml" + + def __init__(self, source_fs: AbstractFileSystem): + """ + Initialize PayloadExtractor and auto-detect the root. + + We expect the top level directory to have a ``package.toml`` file, an + ``entities`` directory, and a ``collections`` directory. A common error + when creating an archive is to zip up the parent directory instead. This + gives us a root that looks like:: + + some-folder-name/package.toml + some-folder-name/entities + some-folder-name/collections + + This is simple to check for and handle, so we just accept this kind of + input by wrapping the original source filesystem with a + ``DirFileSystem`` initialized with whatever the root path should be. + + We wrap unconditionally, even when the archive is laid out correctly, so + that ``self.fs.path`` is a consistent answer to "what did we treat as + the root?" -- and so that we don't have to store that answer separately. + This still works even if ``source_fs`` is another ``DirFileSystem``. + """ + # Note: We can't pass path="" because of how DirFileSystem is + # implemented (it will become ``None``, and that will raise errors). + # Instead, we pass path="/" when things are where they should be at the + # top level. Confusingly, DirFileSystem will normalize this to "", so + # if you later query for it, expect self.fs.path == "" for this case. + self.fs = DirFileSystem( + path=self.find_archive_root(source_fs), fs=source_fs + ) + + @classmethod + def find_archive_root(cls, source_fs: AbstractFileSystem) -> str: + """ + Find the folder to treat as the archive root. + + Returns "/" if the archive's contents are already at the top level, i.e. + there's no wrapper folder to descend into. + + People often build an archive by compressing a folder rather than that + folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a + single top-level directory with everything (including package.toml) + inside it. That is a reasonable thing to hand us, so we accept it. + + We only look one level down, and we require that the candidate directory + actually contains a ``ROOT_PACKAGE_PATH``. + + This function never raises. An archive with no package.toml anywhere + returns "/", and the missing file is reported later as an extraction + error, which is where that error belongs. Note that both DirFileSystem + and ZipFileSystem normalize "/" to "", which makes the wrapper a + pass-through. + """ + if source_fs.exists(cls.ROOT_PACKAGE_PATH): + return "/" + + candidates = [ + entry + for entry in source_fs.ls("/", detail=False) + if source_fs.isdir(entry) and source_fs.exists(f"{entry}/{cls.ROOT_PACKAGE_PATH}") + ] + + # More than one candidate is ambiguous, and guessing would be worse than + # saying we couldn't find the file. + if len(candidates) == 1: + return candidates[0] + + # If we get here, the archive is broken, but we return the top-level + # anyway and let other error-handling code flag that package.toml is + # missing. + return "/" + + def extract(self) -> UnvalidatedLearningPackageInput: + """ + Read the whole archive, gathering errors rather than raising them. + + The general philosophy here is to always march on and get as much as + possible, even if we know the upload is doomed. It's better to see all + the errors for an archive at once, than to see only one error and have + to go through the upload cycle to see the next issue. + """ + # The ``unvalidated`` dict will eventually be used to load a + # ``CompletePackageInputData``, so it needs to match its fields: + # ``meta``, ``learning_package``, ``entities``, and ``collections``. + unvalidated: dict = {} + errors: list[ExtractionError] = [] + + # Root Package Metadata + try: + # This adds the "meta" and "learning_package" keys + unvalidated |= self.extract_root_package_data() + except ExtractionError as err: + errors.append(err) + + # PublishableEntities & versions (components, units, sections, subsections) + entities_data, entities_errors = self.extract_entities_data( + self.get_entity_file_paths() + ) + unvalidated["entities"] = entities_data + errors.extend(entities_errors) + + # Collections. Note that duplicate Collection keys are *not* checked + # here: unlike entities, collections are assembled into a list, so a + # duplicate loses no data at this layer. It's caught during validation + # by CompletePackageInputData.check_for_duplicate_keys, which can give a + # better message because it has both files' data by then. + collections = [] + for collection_file_path in self.get_collection_file_paths(): + try: + collections.append(self.extract_collection_data(collection_file_path)) + except ExtractionError as err: + errors.append(err) + unvalidated["collections"] = collections + + return UnvalidatedLearningPackageInput( + raw_data=unvalidated, + errors=errors, + fs=self.fs, + ) + + def extract_root_package_data(self, path: str | None = None) -> dict: + """ + Extract the "meta" and "learning_package" from the TOML file at path. + + This is a straightforward extraction because we don't have to transform + the actual fields in the data. We expect to see a TOML file that looks + something like this: + + [meta] + format_version = 1 + created_by = "eddy" + created_by_email = "eddy@axim.org" + created_at = 2026-03-11T19:20:20.394360Z + origin_server = "studio.local.openedx.io" + + [learning_package] + title = "Fun Library" + key = "lib:Axim:FunLib" + description = "My very fun library! 🐢" + created = 2026-02-11T16:32:47.524556Z + updated = 2026-02-20T16:32:47.524556Z + + The output should look like: + + { + 'meta': { + 'format_version': 1, + 'created_by': 'eddy', + 'created_by_email': 'eddy@axim.org', + 'created_at': datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), + 'origin_server': 'studio.local.openedx.io' + }, + 'learning_package': { + 'title': 'Fun Library', + 'key': 'lib:Axim:FunLib', + 'description': 'My very fun library! 🐢', + 'created': datetime(2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc), + 'updated': datetime(2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc), + } + } + + We need to return a Python dict that we get from parsing this. Most of + this method is error handling. The error checking at this layer is + minimal, and is mostly focused on making sure that the file exists, is + parseable, and has the two tables we expect it to have. + + Note that in any real world usage, we're just reading the default + ``ROOT_PACKAGE_PATH``. Passing the ``path`` explicitly just makes it + easier to have test package files with descriptive names for debugging. + """ + file_description = "Root Package" + if path is None: + path = self.ROOT_PACKAGE_PATH + + # Check: Root Package file exists at all. + if not self.fs.exists(path): + raise MissingFileError(file_description, path=path) + + root_package_dict = self._load_toml(path, file_description) + + # Check: Don't allow top-level fields outside a [table] + self._check_all_fields_in_tables(root_package_dict, file_description, path) + + # Check: The "[meta]" and "[learning_package]" tables are mandatory + if "meta" not in root_package_dict: + raise TableNotFoundError(file_description, table="meta", path=path) + if "learning_package" not in root_package_dict: + raise TableNotFoundError( + file_description, table="learning_package", path=path + ) + + # Check: We only support format_version 1, and don't know what to do + # with anything higher. This leaves us some wiggle-room to declare a 1.x + # version that is backwards compatible, i.e. it will reject 2 and + # higher, but accept 1.1, 1.2, etc. + format_version = root_package_dict["meta"].get("format_version") + is_number = isinstance(format_version, (int, float)) and not isinstance( + format_version, bool + ) + if not is_number or format_version >= 2: + raise UnsupportedFormatError( + f"Format version {format_version} is unsupported (only 1 is supported).", + path=path, + ) + + return root_package_dict + + def get_entity_file_paths(self) -> list[str]: + """ + Find all the PublishableEntity TOML file paths in our archive. + + We expect our entity TOML files to be in the entities directory, but we + have two categories right now: + + * Component TOML: entities/xblock.v1/{component_type}/{component_code} + * Container TOML: entities/{entity_ref} + + This method looks for TOML files in entities/ or any of its subdirs. We + only exclude matches inside the component_version data, to make sure + that we don't accidentally match media files in the unlikely event where + people have TOML files as static assets. + """ + paths = [ + path + for path in self.fs.glob("entities/**/*.toml") + # Filter out TOML files that are in component media, e.g. static assets: + if "/component_versions/" not in path + ] + return sorted(paths) # Make the ordering deterministic. + + def get_collection_file_paths(self) -> list[str]: + """ + Find all the Collection TOML file paths in our archive. + """ + return sorted(self.fs.glob("collections/*.toml")) + + def extract_entities_data(self, paths: list[str]): + """ + Extract every entity file, collecting errors instead of raising them. + + Returns an ``(entities_data, errors)`` tuple, where entities_data is a + list in the same order as ``paths``. + + Note that we don't check for duplicate entity keys here. Each entity + records the file it came from, and they're kept in a list rather than + being collapsed into a dict, so a redefinition survives this stage + intact for validation to reject. That's the same division of labour we + use for Collections. + """ + entities_data: list[dict] = [] + errors: list[ExtractionError] = [] + for entity_file_path in paths: + try: + entities_data.append(self.extract_entity_data(entity_file_path)) + except ExtractionError as err: + errors.append(err) + + return entities_data, errors + + def extract_entity_data(self, path: str) -> dict: + """ + This extracts raw entity data from an Entity TOML file. + + PublishableEntities can be both Components (XBlock problems, videos, + etc.), as well as Containers like Units, Subsections, and Sections. Some + sample TOML: + + [entity] + can_stand_alone = true + key = "section-9-ac4b9f" + created = 2026-04-08T15:22:12.780012Z + + [entity.draft] + version_num = 2 + + [entity.published] + version_num = 1 + + [entity.container.section] + + # ### Versions + + [[version]] + title = "Section 9" + version_num = 2 + + [version.container] + children = ["week-7-e73782", "subsection-001-e4bbe5"] + + [[version]] + title = "Section 9" + version_num = 1 + + [version.container] + children = ["week-7-e73782"] + + We return a tuple where the first element is the Entity's key + ("section-9-ac4b9f"), and the second is a dict that would look like: + + { + 'can_stand_alone': True, + 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), + 'draft': { + 'version_num': 2 + }, + 'published': { + 'version_num': 1 + }, + 'container': { + 'section': {} + }, + 'versions': [ + { + 'title': 'Section 9', + 'version_num': 2, + 'container': { + 'children': [ + 'week-7-e73782', + 'subsection-001-e4bbe5' + ] + } + }, + { + 'title': 'Section 9', + 'version_num': 1, + 'container': { + 'children': [ + 'week-7-e73782' + ] + } + } + ] + } + + Note some key differences: + + 1. The "entity" table elements have been popped out to the top level. + 2. The "version" list has been renamed to "versions" to feel more + natural. + 3. A "src_path" has been added, recording the file this came from, so + that validation can name it in an error message. + """ + file_description = "Entity" + + entity_root_dict = self._load_toml(path, file_description) + + # Check: Don't allow top-level fields outside a [table] + self._check_all_fields_in_tables(entity_root_dict, file_description, path) + + # Check: Does it define a top level "[entity]" table? Note that this can + # pass if they define a sub-table like "[entity.draft]", since the + # existence of "[entity]" is implicit in that case. If we get that far, + # rely on catching it at the validation step (i.e. after payload + # extraction). + if "entity" not in entity_root_dict: + raise TableNotFoundError(file_description, "entity", path=path) + + # Note that we don't check for a missing "key" here. It's a required + # field on EntityInputData, so validation reports it, in the same way it + # does for a Collection with no key. + entity = entity_root_dict["entity"] + entity["src_path"] = path + + # Note case difference: we're renaming "version" in the TOML to + # "versions" in the data dict we're assembling. + entity["versions"] = entity_root_dict.pop("version", []) + for version in entity["versions"]: + self._add_component_version_media(version, path) + + return entity + + def extract_collection_data(self, path: str) -> dict: + """ + Extract the contents of a single Collection TOML file. + + We record the source path on the way out, so that a later duplicate-key + error can name both of the files involved. + """ + file_description = "Collection" + + collection_root_dict = self._load_toml(path, file_description) + + self._check_all_fields_in_tables(collection_root_dict, file_description, path) + if "collection" not in collection_root_dict: + raise TableNotFoundError(file_description, table="collection", path=path) + + collection_data = collection_root_dict["collection"] + collection_data["src_path"] = path + + return collection_data + + def _add_component_version_media(self, version: dict, entity_path: str) -> None: + """ + Attach a Component version's media, if this version has any on disk. + + Do our best to put together entity version data (and component version + data), but don't worry about validating the results (that can happen + during the validation step). + """ + version_num = version.get("version_num") + comp_ver_dir = os.path.join( + os.path.splitext(entity_path)[0], + "component_versions", + f"v{version_num}", + ) + if not self.fs.exists(comp_ver_dir): + return + + media = { + os.path.relpath(media_path, comp_ver_dir): self.fs.read_text(media_path) + for media_path in self.fs.glob(f"{comp_ver_dir}/*") + if self.fs.isfile(media_path) + } + # Any static files are encoded as pointers. + # TODO: Add support for data-urls + for static_file_path in self.fs.glob(f"{comp_ver_dir}/static/**"): + if self.fs.isfile(static_file_path): + rel_path = os.path.relpath(static_file_path, comp_ver_dir) + media[rel_path] = static_file_path + + version["component"] = {"media": media} + + def _load_toml(self, path: str, file_description: str) -> dict: + """ + Parse the TOML file at ``path``, or raise InvalidTOMLError. + """ + with self.fs.open(path, "rb") as toml_file: + try: + return tomllib.load(toml_file) + except tomllib.TOMLDecodeError as dec_err: + raise InvalidTOMLError( + file_description, details=str(dec_err), path=path + ) from dec_err + + @staticmethod + def _check_all_fields_in_tables(data: dict, file_description: str, path: str): + """ + Raise an error if fields are declared outside of a table. + + The convention for our TOML files is that keys are always in a table, so if + it's *not* in a table, that's likely an omission/error that might otherwise + be difficult to catch because they'd be "missing" from the place they're + supposed to be in the parsed data structure, but that wouldn't be obvious to + someone editing the files by hand. + """ + fields_outside_of_tables = [ + field + for field, val in data.items() + if not isinstance(val, dict) and not isinstance(val, list) + ] + if fields_outside_of_tables: + raise FieldsNotInTable( + file_description, fields=fields_outside_of_tables, path=path + ) diff --git a/src/openedx_content/applets/backup_restore/readme.rst b/src/openedx_content/applets/backup_restore/readme.rst new file mode 100644 index 000000000..e9b33abfc --- /dev/null +++ b/src/openedx_content/applets/backup_restore/readme.rst @@ -0,0 +1,13 @@ +Backup/Restore Applet +===================== + +The ``backup_restore`` applet is responsible for making a backup archive of an existing Learning Package, or creating a new Learning Package from an existing archive. + +Restoring (reading an archive) is done with pydantic schema validation, in a pipeline of small modules that hand plain data to each other:: + + archive.py Archive location (a path) → FileSystem (fsspec) + payload.py FileSystem (fsspec) → UnvalidatedLearningPackageInput + validation.py UnvalidatedLearningPackageInput → ValidatedLearningPackageInput + loading.py ValidatedLearningPackageInput → LearningPackage + +Backing up (writing an archive) has not been migrated yet. It still lives in ``zipper.py`` and ``toml.py``, which build TOML directly from the Django models. The ``*OutputData`` models in ``schema.py`` are the beginning of that work, but nothing uses them yet. diff --git a/src/openedx_content/applets/backup_restore/results.py b/src/openedx_content/applets/backup_restore/results.py new file mode 100644 index 000000000..04f872661 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/results.py @@ -0,0 +1,111 @@ +""" +The values we hand back after restoring a Learning Package from an archive. + +These live in their own module (rather than next to either the reading or the +writing code) because both the archive format and the database models change +independently of the summary we report to callers. + +TODO: ``RestoreResult`` and friends are shaped by what the frontend currently +expects, not by what's natural here. When we revisit the REST API, the loader +should return something structured and let ``api.py`` do the translation. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass +from datetime import datetime +from io import StringIO +from typing import Literal + +from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user + + +@dataclass +class RestoreLearningPackageData: + """ + Data about the restored learning package. + """ + id: int # The ID of the restored learning package + package_ref: str # The package_ref of the restored learning package (may be different if staged) + archive_package_ref: str # The original package_ref from the archive + archive_org_code: str | None # The org code parsed from archive_package_ref, or None if unparseable + archive_package_code: str | None # The package code parsed from archive_package_ref, or None if unparseable + title: str + num_containers: int + num_sections: int + num_subsections: int + num_units: int + num_components: int + num_collections: int + + +@dataclass +class BackupMetadata: + """ + Metadata about the backup operation. + """ + format_version: int + created_at: datetime | str | None + created_by: str | None = None + created_by_email: str | None = None + original_server: str | None = None + + +@dataclass +class RestoreResult: + """ + Result of the restore operation. + """ + status: Literal["success", "error"] + log_file_error: StringIO | None = None + lp_restored_data: RestoreLearningPackageData | None = None + backup_metadata: BackupMetadata | None = None + + +def unpack_package_ref(package_ref: str) -> tuple[str | None, str | None]: + """ + Try to parse org_code and package_code from a package_ref. + + By convention, package_refs take the form ``"{prefix}:{org_code}:{package_code}"``, + but this is only a convention — package_ref is opaque and the parse may fail. + Returns ``(None, None)`` if the ref does not match the expected format. + """ + parts = package_ref.split(":") + if len(parts) < 3: + return None, None + _, org_code, package_code = parts[:3] + return org_code, package_code + + +def generate_staged_package_ref(archive_package_ref: str, user: UserType) -> str: + """ + Generate a staged learning package ref based on the archive's package_ref. + + We can't trust package_ref from the archive directly, because the archive + could specify *any* arbitrary package_ref, and the user may or may not be + permitted to create an Package using that ref. So, instead, this function + generates a unique and semi-human-readable package_ref which is namespaced + to the current user and appropriate to provisionally save the package under. + The package_ref from the archive can then be presented to the user as a + *suggestion*, which they may or may not choose to use. + + Please note that the ref returned by this function is valid for Packages is + a generic sense, but it's not a valid Content Library key. Callers who are + restoring a Package for Library usage will need to replace this staged + package_ref before being able to render the Library's content. + + Arguments: + archive_package_ref (str): The original package_ref from the archive. + user (UserType | None): The user performing the restore operation. + + Example: + Input: "lib:WGU:LIB_C001" + Output: "lp-restore:dave:WGU:LIB_C001:1728575321" + """ + username = user.username + org_code, package_code = unpack_package_ref(archive_package_ref) + timestamp = int(time.time() * 1000) # Current time in milliseconds + if org_code and package_code: + return f"lp-restore:{username}:{org_code}:{package_code}:{timestamp}" + # Fallback for non-conventional package_refs + return f"lp-restore:{username}:{archive_package_ref}:{timestamp}" diff --git a/src/openedx_content/applets/backup_restore/schema.py b/src/openedx_content/applets/backup_restore/schema.py new file mode 100644 index 000000000..23d02bb5e --- /dev/null +++ b/src/openedx_content/applets/backup_restore/schema.py @@ -0,0 +1,292 @@ +""" +This module defines the schema that we use during the backup/restore process. + +The pydantic models defined in this module are divided into InputData and +OutputData. These are intentionally kept separate and do not inherit from each +other. The InputData classes will be much more permissive, with many optional +fields. The OutputData classes are meant for internal use when generating +exports, and will be stricter. +""" +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import ( + AwareDatetime, + BaseModel, + ConfigDict, + EmailStr, + Field, + StrictStr, + StringConstraints, + field_validator, +) + +# Refs are arbitrary identifiers that we do almost no validation of, and are +# mainly there to assure uniqueness within some namespace. +REF_CONSTRAINTS = StringConstraints( + strict=True, + strip_whitespace=True, +) + +# This is for things like the collection_code, library_code, etc. +CODE_CONSTRAINTS = StringConstraints( + strict=True, + strip_whitespace=True, + # Note that we can't use \Z to indicate the end of line in our regex because + # that's not supported syntax in JavaScript, and pydantic will raise an + # error when trying to generate a JSON Schema. However, the combination of $ + # and strip_whitespace=True means that we're sure that we won't allow any + # trailing newlines. + pattern=r"^[a-zA-Z0-9_.-]+$", +) + + +def _reject_duplicate_keys(items, label: str) -> None: + """ + Raise a ValueError if two items in ``items`` share a ``key``. + + Both Entities and Collections need this, and the message wants to name the + file that redefined the key as well as the one that got there first, which is + what each item's ``src_path`` is for. + """ + keys_to_items: dict[str, object] = {} + for item in items: + if item.key in keys_to_items: + original = keys_to_items[item.key] + raise ValueError( + f'{label} "{item.key}" redefined in ' + f'{item.src_path} (original in ' + f'{original.src_path})' # type: ignore[attr-defined] + ) + keys_to_items[item.key] = item + + +class InputData(BaseModel): + """ + Base class for all inputs, here to set config defaults. + + InputData classes are frozen, i.e. they should only be initialized once from + the unvalidated input. Allowing gradual mutations makes things much harder + to debug. + + InputData clases are also set to allow parameters that they don't recognize + (extra="allow") for the sake of forwards compatibility. As any given file + format gets iterated on, it will get new attributes. Older installs of the + platform should ignore these new attributes and just load the things that we + know how to handle. The reason we don't set this to "ignore" is because + unrecognized fields could be simple typos of known fields, so we still want + to capture that information so we can potentially display warnings about it. + """ + model_config = ConfigDict(frozen=True, extra="allow") + + +class CompletePackageInputData(InputData): + """ + The contents of the entire Learning Package. + """ + meta: MetaInputData + learning_package: LearningPackageInputData + + # These are lists rather than dicts keyed by their "key" field, and that is + # deliberate: a dict can only hold one entry per key, so a duplicated key + # would silently overwrite its predecessor during extraction and there would + # be nothing left for us to complain about here. Keeping them as lists is + # what lets the duplicate checks below exist at all. + entities: list[EntityInputData] + + collections: list[CollectionInput] + + @field_validator('entities', mode='after') + @classmethod + def check_for_duplicate_entity_keys(cls, entities: list[EntityInputData]): + """ + Raise a ValueError if the same Entity is defined in two places. + """ + _reject_duplicate_keys(entities, "Entity") + return entities + + @field_validator('collections', mode='after') + @classmethod + def check_for_duplicate_collection_keys(cls, collections: list[CollectionInput]): + """ + Raise a ValueError if we encounter a duplicate collection entry. + + In the longer term, we may want to be able to remove the duplicate + entries (and other broken entries), while still otherwise allowing the + restore to proceed. But for now, any error kills the restore process. + """ + _reject_duplicate_keys(collections, "Collection") + return collections + + +class MetaInputData(InputData): + """ + Input Package Metadata, Version 1 + + This is data about the backup file itself, as opposed to the Learning + Package that it contains: who created this backup, when was it created, etc. + On the input side, the fields here are only here so that we can give useful + preview information when the user is uploading this to a new instance. None + of these values are necessary for creating a new Learning Package—in fact, + none of these can even be trusted, since a malicious actor could manipulate + them to say whatever they wanted. It's just meant as a sanity check to help + assure the user that they're restoring the correct package archive. + + The only truly critical field is ``format_version``, since that will one day + affect input validation rules. + """ + format_version: Literal[1] # Only supported version at the moment + created_by: StrictStr | None = Field(default=None, min_length=1) + created_by_email: EmailStr | None = None + created_at: AwareDatetime | None = None + origin_server: StrictStr | None = None + + @field_validator("created_by", "created_by_email", "origin_server", mode="before") + @classmethod + def blank_means_absent(cls, value): + """ + Treat a blank string in [meta] as "not supplied". + + The backup side writes these fields unconditionally, so an archive made + by a user with no email address on file carries + ``created_by_email = ""``. Since none of this metadata is required (or + even trustworthy), refusing to restore such an archive would be far + worse than not knowing who made it. + """ + if isinstance(value, str) and not value.strip(): + return None + return value + + +class LearningPackageInputData(InputData): + """ + High level data for a Learning Package itself (not its contents). + """ + title: StrictStr = Field(min_length=1, default="Untitled Library") + key: Annotated[ + str, + REF_CONSTRAINTS, + Field( + description=( + "This is often a LibraryLocatorV2-formatted string, but can be " + "any arbitrary string at the moment. It must be unique within a" + " given server instance." + ), + examples=[ + "lib:OrgName:LibraryName", + "lib:Axim:IntroPhysics", + "lp-restore:Axim:IntroPhysics:1775752130941", + ], + ), + ] + description: StrictStr | None = Field(default="", max_length=10_000) + created: AwareDatetime | None = None + updated: AwareDatetime | None = None + + +class DraftInputData(InputData): + version_num: Annotated[int, Field(gt=0)] | None = None + + +class PublishedInputData(InputData): + version_num: Annotated[int, Field(gt=0)] | None = None + + +class EntityInputData(InputData): + """A PublishableEntity: either a Component or a Container.""" + + key: Annotated[str, REF_CONSTRAINTS] + + can_stand_alone: bool = True + + created: AwareDatetime + + # Weird edge case: If you create something, never publish it, and then do a + # "reset to published state", the resulting export in Ulmo would omit the + # [entity.draft] section entirely, rather than it being an empty dictionary. + draft: DraftInputData = DraftInputData(version_num=None) + published: PublishedInputData = PublishedInputData(version_num=None) + + versions: list[VersionInput] = [] + + # Not all entities are containers, and we may one day have containers that + # this version of the code does not understand. So we have a generic dict + # for unknown containers and None means it's something that is not a + # container. + # + # TODO: Test unknown container type. + container: UnitInputData | SubsectionInputData | SectionInputData | dict | None = None + + # The source file this Entity was defined in. See the note on + # CollectionInput.src_path -- this is for error messages only. + src_path: str | None = None + + +class SectionInputData(InputData): + """Marks an entity as a Section.""" + + # Note: this field is intentionally required, with no default. These three + # container models are all `extra="allow"`, so if the discriminating field + # were optional, every one of them would happily validate every dict and the + # union in EntityInputData.container would always resolve to whichever model + # is listed first. + section: dict + + +class SubsectionInputData(InputData): + subsection: dict # Required. See the note on SectionInputData. + + +class UnitInputData(InputData): + unit: dict # Required. See the note on SectionInputData. + + +class VersionInput(InputData): + version_num: Annotated[int, Field(gt=0)] + title: str + component: ComponentVersionInput | None = None + container: ContainerVersionInput | None = None + + +class ComponentVersionInput(InputData): + media: dict + + +class ContainerVersionInput(InputData): + children: list[Annotated[str, REF_CONSTRAINTS]] + + +class CollectionInput(InputData): + """A named grouping of PublishableEntities within a Learning Package.""" + + title: StrictStr = Field(min_length=1) + key: Annotated[ + str, + CODE_CONSTRAINTS, + Field( + description=( + "A unique slug-like code field. Must be unique within a given Learning Package." + ), + examples=[ + "difficult-problems", + "practice-exams", + ], + ), + ] + description: StrictStr | None = Field(default="", max_length=10_000) + + # It looks like we weren't actually serializing the modified date. + created: AwareDatetime | None = None + + # The PublishableEntities that belong to this Collection. Entity refs that + # aren't in the archive are ignored at load time rather than being an error, + # since a Collection with a dangling member is still perfectly usable. + entities: list[Annotated[str, REF_CONSTRAINTS]] = [] + + # This is the source file where this Collection was defined. This is only + # for being able to create useful error messages. We should never be reading + # from this file directly because the exact format of this file should be + # free to change as needed. That's the responsibilty of the payload.py + # module. + src_path: str | None = None diff --git a/src/openedx_content/applets/backup_restore/serializers.py b/src/openedx_content/applets/backup_restore/serializers.py deleted file mode 100644 index 105c25094..000000000 --- a/src/openedx_content/applets/backup_restore/serializers.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -The serializers module for restoration of authoring data. - -Please note that the serializers are defined from the perspective of the -TOML format, with the models as the "source". That is, when the model fields -and TOML fields differ, we'll declare it like this: - - my_toml_field = serializers.BlahField(source="my_model_field") -""" -from datetime import timezone - -from rest_framework import serializers - -from ..components import api as components_api -from ..components.models import ComponentType - - -class LearningPackageSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for learning packages. - - Note: - The ref/key field is serialized but is generally not trustworthy for - restoration. During restore, a new ref may be generated or overridden. - """ - - title = serializers.CharField(required=True) - # The model field is now LearningPackage.package_ref, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="package_ref") - description = serializers.CharField(required=True, allow_blank=True) - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - - -class LearningPackageMetadataSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for learning package metadata. - - Note: - This serializer handles data exported to an archive (e.g., during backup), - but the metadata is not restored to the database and is meant solely for inspection. - """ - format_version = serializers.IntegerField(required=True) - created_by = serializers.CharField(required=False, allow_null=True) - created_by_email = serializers.EmailField(required=False, allow_null=True) - created_at = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - origin_server = serializers.CharField(required=False, allow_null=True) - - -class EntitySerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for publishable entities. - """ - - can_stand_alone = serializers.BooleanField(required=True) - # The model field is now PublishableEntity.entity_ref, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="entity_ref") - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - - -class EntityVersionSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for publishable entity versions. - """ - # We allow_blank because empty unit titles are legal and common. - title = serializers.CharField(required=True, allow_blank=True) - - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - version_num = serializers.IntegerField(required=True) - - # Note: Unlike the fields above, `entity_ref` does not appear on the model - # nor in the TOML. This is just added by the validation pipeline for convenience. - entity_ref = serializers.CharField(required=True) - - -class ComponentSerializer(EntitySerializer): # pylint: disable=abstract-method - """ - Serializer for components. - Contains logic to convert entity_key to component_type and component_code. - """ - - def validate(self, attrs): - """ - Custom validation logic: - parse the entity_key into (component_type, component_code). - """ - entity_key = attrs["entity_ref"] - try: - component_type_obj, component_code = _get_or_create_component_type_by_entity_key(entity_key) - attrs["component_type"] = component_type_obj - attrs["component_code"] = component_code - except ValueError as exc: - raise serializers.ValidationError({"key": str(exc)}) - return attrs - - -def _get_or_create_component_type_by_entity_key(entity_key: str) -> tuple[ComponentType, str]: - """ - Get or create a ComponentType based on a full [entity].key string. - - The entity key is expected to be in the format - ``"{namespace}:{type_name}:{component_code}"``. This function will parse out - the ``namespace`` and ``type_name`` parts and use those to get or create the - ComponentType. - - Raises ValueError if the entity_key is not in the expected format. - - Historical note: In Ulmo, this function was part of the public API. This was - inappropriate because the exact format of entity_keys is just a convention - rather than something API callers should count on. That said, it is safe to - assume that in all "v1" archives, the components' entity keys are safe to - parse into (namespace, type, code). So, we have moved this parsing logic - from the public API to just this internal halper function. Future devs, - please do not make new external guarantees about the format of entity keys - (aka entity_refs). A future "v2" backup-restore format will drop this - assumption of parse-ability.. - """ - try: - namespace, type_name, component_code = entity_key.split(':', 2) - except ValueError as exc: - raise ValueError( - f"Invalid entity_key format: {entity_key!r}. " - "Expected format: '{namespace}:{type_name}:{component_code}'" - ) from exc - return components_api.get_or_create_component_type(namespace, type_name), component_code - - -class ComponentVersionSerializer(EntityVersionSerializer): # pylint: disable=abstract-method - """ - Serializer for component versions. - """ - - -class ContainerSerializer(EntitySerializer): # pylint: disable=abstract-method - """ - Serializer for containers. - """ - container = serializers.DictField(required=True) - - def validate_container(self, value): - """ - Custom validation logic for the container field. - Ensures that the container dict has exactly one key which is one of - "section", "subsection", or "unit" values. - """ - errors = [] - if not isinstance(value, dict) or len(value) != 1: - errors.append("Container must be a dict with exactly one key.") - if len(value) == 1: # Only check the key if there is exactly one - container_type = list(value.keys())[0] - if container_type not in ("section", "subsection", "unit"): - errors.append(f"Invalid container value: {container_type}") - if errors: - raise serializers.ValidationError(errors) - return value - - def validate(self, attrs): - """ - Custom validation logic: - parse the container dict to extract the container type. - """ - container = attrs["container"] - container_type = list(container.keys())[0] # It is safe to do this after validate_container - attrs["container_type"] = container_type - attrs.pop("container") # Remove the container field after processing - return attrs - - -class ContainerVersionSerializer(EntityVersionSerializer): # pylint: disable=abstract-method - """ - Serializer for container versions. - """ - container = serializers.DictField(required=True) - - def validate_container(self, value): - """ - Custom validation logic for the container field. - Ensures that the container dict has exactly one key "children" which is a list of strings. - """ - errors = [] - if not isinstance(value, dict) or len(value) != 1: - errors.append("Container must be a dict with exactly one key.") - if "children" not in value: - errors.append("Container must have a 'children' key.") - if "children" in value and not isinstance(value["children"], list): - errors.append("'children' must be a list.") - if errors: - raise serializers.ValidationError(errors) - return value - - def validate(self, attrs): - """ - Custom validation logic: - parse the container dict to extract the children list. - """ - children = attrs["container"]["children"] # It is safe to do this after validate_container - attrs["children"] = children - attrs.pop("container") # Remove the container field after processing - return attrs - - -class CollectionSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for collections. - """ - title = serializers.CharField(required=True) - # The model field is now Collection.collection_code, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="collection_code") - description = serializers.CharField(required=True, allow_blank=True) - entities = serializers.ListField( - child=serializers.CharField(), - required=True, - allow_empty=True, - ) diff --git a/src/openedx_content/applets/backup_restore/toml.py b/src/openedx_content/applets/backup_restore/toml.py index b5afa6174..e1fcff3bd 100644 --- a/src/openedx_content/applets/backup_restore/toml.py +++ b/src/openedx_content/applets/backup_restore/toml.py @@ -3,7 +3,6 @@ """ from datetime import datetime -from typing import Any, Dict import tomlkit from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user @@ -233,27 +232,3 @@ def toml_collection(collection: Collection, entity_refs: list[str]) -> str: doc.add("collection", collection_table) return tomlkit.dumps(doc) - - -def parse_learning_package_toml(content: str) -> dict: - """ - Parse the learning package TOML content and return a dict of its fields. - """ - lp_data: Dict[str, Any] = tomlkit.parse(content) - return lp_data - - -def parse_publishable_entity_toml(content: str) -> dict: - """ - Parse the publishable entity TOML file and return a dict of its fields. - """ - pe_data: Dict[str, Any] = tomlkit.parse(content) - return pe_data - - -def parse_collection_toml(content: str) -> dict: - """ - Parse the collection TOML content and return a dict of its fields. - """ - collection_data: Dict[str, Any] = tomlkit.parse(content) - return collection_data diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py new file mode 100644 index 000000000..d838fa1c9 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -0,0 +1,219 @@ +""" +Archive-agnostic validation of the input data models. + +This sits between ``payload.py`` (which knows about files) and ``loading.py`` +(which knows about the database). It answers one question: is this input good +enough to load? It never raises -- every problem it finds is collected onto +``ValidatedLearningPackageInput.errors`` so that someone repairing an archive by +hand can see everything that's wrong in one pass. + +There are three sources of errors here: + +1. Errors that ``payload.py`` already found while reading the files. Those are + ``ExtractionError`` instances and are carried through as-is. +2. Schema errors, from running the raw data through ``CompletePackageInputData``. + Pydantic reports these against a path into the assembled document, so we + translate that back into the archive file it came from. +3. Consistency errors -- cross-references that pydantic has no way to express, + like a container listing a child that isn't anywhere in the archive. +""" +from __future__ import annotations + +import attrs +from fsspec.implementations.dirfs import DirFileSystem +from pydantic import ValidationError + +from .errors import ( + BackupRestoreError, + DuplicateVersionError, + MalformedRefError, + MissingVersionError, + SchemaError, + UnknownContainerTypeError, + UnresolvedChildError, +) +from .payload import PayloadExtractor, UnvalidatedLearningPackageInput +from .schema import CompletePackageInputData + + +@attrs.define(frozen=True) +class ValidatedLearningPackageInput: + """ + The result of validating an archive's contents. + + ``data`` is ``None`` when the input was too broken to build a model from at + all. ``errors`` being non-empty means the restore must not proceed, even if + ``data`` is populated -- a consistency error can be found on a document that + is otherwise structurally valid. + """ + + data: CompletePackageInputData | None + + # The re-rooted filesystem from extraction. ``fs.path`` is the folder inside + # the archive that was treated as its root, or "" if there wasn't one. + fs: DirFileSystem + + errors: list[BackupRestoreError] + + +def validate( + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> ValidatedLearningPackageInput: + """ + Validate extracted archive data, gathering every error we can find. + """ + # Extraction errors are already BackupRestoreErrors, so they just come along. + errors: list[BackupRestoreError] = list(unvalidated_lp.errors) + + try: + data = CompletePackageInputData.model_validate(unvalidated_lp.raw_data) + except ValidationError as val_err: + data = None + errors.extend(_schema_errors_for(val_err, unvalidated_lp)) + + if data is not None: + errors.extend(_consistency_errors_for(data)) + + return ValidatedLearningPackageInput( + data=data, + fs=unvalidated_lp.fs, + errors=errors, + ) + + +def _schema_errors_for( + val_err: ValidationError, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> list[SchemaError]: + """ + Turn one pydantic ValidationError into one SchemaError per problem found. + """ + return [ + SchemaError( + message=entry["msg"], + **_source_for_loc(entry["loc"], unvalidated_lp), + ) + for entry in val_err.errors() + ] + + +def _source_for_loc( + loc: tuple, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> dict: + """ + Map a pydantic ``loc`` back to the archive file it came from. + + Pydantic reports errors against the combined document we assemble in + ``payload.py``, e.g. ``("entities", 3, "versions", 0, "title")``. Nobody + editing an archive has ever seen that document, so we split the ``loc`` into + the file it came from and the location within that file. + """ + match loc: + case ("entities" | "collections" as section, int() as index, *rest): + return { + "path": _src_path_at(section, index, unvalidated_lp), + "location": tuple(rest), + } + case ("meta" | "learning_package", *_): + # TODO: This is a problematic abstraction leak + return {"path": PayloadExtractor.ROOT_PACKAGE_PATH, "location": tuple(loc)} + + return {"path": None, "location": tuple(loc)} + + +def _src_path_at( + section: str, + index: int, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> str | None: + """ + Look up the source file of the item at ``index`` of ``section``. + + We can't read this off the validated model, because we only need it when + validation has already failed. + """ + raw_items = unvalidated_lp.raw_data.get(section, []) + if 0 <= index < len(raw_items): + raw_item = raw_items[index] + if isinstance(raw_item, dict): + return raw_item.get("src_path") + return None + + +def _consistency_errors_for( + data: CompletePackageInputData, +) -> list[BackupRestoreError]: + """ + Check the cross-references that pydantic can't express. + + Every check here corresponds to something that would otherwise blow up in the + middle of ``loading.py``, with a traceback pointing at our code instead of at + the part of the archive that's actually wrong. + """ + errors: list[BackupRestoreError] = [] + known_refs = {entity.key for entity in data.entities} + + for entity in sorted(data.entities, key=lambda e: e.key): + entity_ref = entity.key + path = entity.src_path + + # Check: is this a container type we actually know how to build? + if isinstance(entity.container, dict): + declared = ", ".join(sorted(entity.container)) or "(empty)" + errors.append( + UnknownContainerTypeError( + f'Entity "{entity_ref}" declares an unsupported container ' + f"type: {declared}", + path=path, + ) + ) + elif entity.container is None: + # Not a container, so it's a Component, and we derive the component + # type from the ref itself. + if len(entity_ref.split(":")) != 3: + errors.append( + MalformedRefError( + f'Component ref "{entity_ref}" should be of the form ' + '"{namespace}:{type}:{code}"', + path=path, + ) + ) + + # Check: no version_num declared twice for the same entity. + version_nums = [version.version_num for version in entity.versions] + for duplicated in sorted({v for v in version_nums if version_nums.count(v) > 1}): + errors.append( + DuplicateVersionError( + f'Entity "{entity_ref}" declares version {duplicated} more than once', + path=path, + ) + ) + + # Check: the draft/published pointers name versions that exist here. + available = set(version_nums) + for label, pointer in (("draft", entity.draft), ("published", entity.published)): + if pointer.version_num is not None and pointer.version_num not in available: + errors.append( + MissingVersionError( + f'Entity "{entity_ref}" points [entity.{label}] at version ' + f"{pointer.version_num}, which is not in the archive", + path=path, + ) + ) + + # Check: every child of every container version is in the archive. + for version in entity.versions: + if version.container is None: + continue + for child_ref in version.container.children: + if child_ref not in known_refs: + errors.append( + UnresolvedChildError( + f'Entity "{entity_ref}" v{version.version_num} lists child ' + f'"{child_ref}", which is not defined in the archive', + path=path, + ) + ) + + return errors diff --git a/src/openedx_content/applets/backup_restore/zipper.py b/src/openedx_content/applets/backup_restore/zipper.py index 321cef3e2..b70694e1a 100644 --- a/src/openedx_content/applets/backup_restore/zipper.py +++ b/src/openedx_content/applets/backup_restore/zipper.py @@ -3,25 +3,17 @@ including a TOML representation of the learning package and its entities. """ import hashlib -import time -import tomllib import zipfile -from collections import defaultdict -from dataclasses import asdict, dataclass from datetime import datetime, timezone -from io import StringIO from pathlib import Path -from typing import Any, List, Literal, Optional, Tuple +from typing import List, Optional, Tuple from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user -from django.db import transaction from django.db.models import Prefetch, QuerySet from django.utils.text import slugify -from rest_framework import serializers from openedx_content.models_api import ( Collection, - ComponentType, ComponentVersion, ComponentVersionMedia, LearningPackage, @@ -31,33 +23,10 @@ ) from ..collections import api as collections_api -from ..components import api as components_api -from ..containers import api as containers_api -from ..media import api as media_api from ..publishing import api as publishing_api -from ..sections.models import Section -from ..subsections.models import Subsection -from ..units.models import Unit -from .serializers import ( - CollectionSerializer, - ComponentSerializer, - ComponentVersionSerializer, - ContainerSerializer, - ContainerVersionSerializer, - LearningPackageMetadataSerializer, - LearningPackageSerializer, -) -from .toml import ( - parse_collection_toml, - parse_learning_package_toml, - parse_publishable_entity_toml, - toml_collection, - toml_learning_package, - toml_publishable_entity, -) +from .toml import toml_collection, toml_learning_package, toml_publishable_entity TOML_PACKAGE_NAME = "package.toml" -DEFAULT_USERNAME = "command" def slugify_hashed_filename(identifier: str) -> str: @@ -410,730 +379,3 @@ def create_zip(self, path: str) -> None: toml_collection(collection, list(entity_refs_related)), timestamp=collection.modified, ) - - -@dataclass -class RestoreLearningPackageData: - """ - Data about the restored learning package. - """ - id: int # The ID of the restored learning package - package_ref: str # The package_ref of the restored learning package (may be different if staged) - archive_package_ref: str # The original package_ref from the archive - archive_org_code: str | None # The org code parsed from archive_package_ref, or None if unparseable - archive_package_code: str | None # The package code parsed from archive_package_ref, or None if unparseable - title: str - num_containers: int - num_sections: int - num_subsections: int - num_units: int - num_components: int - num_collections: int - - -@dataclass -class BackupMetadata: - """ - Metadata about the backup operation. - """ - format_version: int - created_at: str - created_by: str | None = None - created_by_email: str | None = None - original_server: str | None = None - - -@dataclass -class RestoreResult: - """ - Result of the restore operation. - """ - status: Literal["success", "error"] - log_file_error: StringIO | None = None - lp_restored_data: RestoreLearningPackageData | None = None - backup_metadata: BackupMetadata | None = None - - -def unpack_package_ref(package_ref: str) -> tuple[str | None, str | None]: - """ - Try to parse org_code and package_code from a package_ref. - - By convention, package_refs take the form ``"{prefix}:{org_code}:{package_code}"``, - but this is only a convention — package_ref is opaque and the parse may fail. - Returns ``(None, None)`` if the ref does not match the expected format. - """ - parts = package_ref.split(":") - if len(parts) < 3: - return None, None - _, org_code, package_code = parts[:3] - return org_code, package_code - - -def generate_staged_package_ref(archive_package_ref: str, user: UserType) -> str: - """ - Generate a staged learning package ref based on the archive's package_ref. - - We can't trust package_ref from the archive directly, because the archive - could specify *any* arbitrary package_ref, and the user may or may not be - permitted to create an Package using that ref. So, instead, this function - generates a unique and semi-human-readable package_ref which is namespaced - to the current user and appropriate to provisionally save the package under. - The package_ref from the archive can then be presented to the user as a - *suggestion*, which they may or may not choose to use. - - Please note that the ref returned by this function is valid for Packages is - a generic sense, but it's not a valid Content Library key. Callers who are - restoring a Package for Library usage will need to replace this staged - package_ref before being able to render the Library's content. - - Arguments: - archive_package_ref (str): The original package_ref from the archive. - user (UserType | None): The user performing the restore operation. - - Example: - Input: "lib:WGU:LIB_C001" - Output: "lp-restore:dave:WGU:LIB_C001:1728575321" - """ - username = user.username - org_code, package_code = unpack_package_ref(archive_package_ref) - timestamp = int(time.time() * 1000) # Current time in milliseconds - if org_code and package_code: - return f"lp-restore:{username}:{org_code}:{package_code}:{timestamp}" - # Fallback for non-conventional package_refs - return f"lp-restore:{username}:{archive_package_ref}:{timestamp}" - - -class LearningPackageUnzipper: - """ - Handles extraction and restoration of learning package data from a zip archive. - - Args: - zipf (zipfile.ZipFile): The zip file containing the learning package data. - user (UserType | None): The user performing the restore operation. Not necessarily the creator. - generate_new_key (bool): Whether to generate a new key for the restored learning package. - - Returns: - dict[str, Any]: The result of the restore operation, including any errors encountered. - - Responsibilities: - - Parse and organize files from the zip structure. - - Restore learning package, containers, components, and collections to the database. - - Ensure atomicity of the restore process. - - Usage: - unzipper = LearningPackageUnzipper(zip_file) - result = unzipper.load() - """ - - def __init__(self, zipf: zipfile.ZipFile, package_ref: str | None = None, user: UserType | None = None): - self.zipf = zipf - self.user = user - self.user_id = getattr(self.user, "id", None) - self.package_ref = package_ref # If provided, use this package_ref for the restored learning package - self.learning_package_id: LearningPackage.ID | None = None # Will be set upon restoration - self.utc_now: datetime = datetime.now(timezone.utc) - self.component_types_cache: dict[tuple[str, str], ComponentType] = {} - self.errors: list[dict[str, Any]] = [] - # Maps for resolving relationships - self.components_map_by_ref: dict[str, Any] = {} - self.units_map_by_ref: dict[str, Any] = {} - self.subsections_map_by_ref: dict[str, Any] = {} - self.sections_map_by_ref: dict[str, Any] = {} - self.all_publishable_entity_refs: set[str] = set() - self.all_published_entities_versions: set[tuple[str, int]] = set() # To track published entity versions - - # -------------------------- - # Public API - # -------------------------- - - @transaction.atomic - def load(self) -> dict[str, Any]: - """Extracts and restores all objects from the ZIP archive in an atomic transaction.""" - - # Step 1: Validate presence of package.toml and basic structure - _, organized_files = self.check_mandatory_files() - if self.errors: - # Early return if preliminary checks fail since mandatory files are missing - result = RestoreResult( - status="error", - log_file_error=self._write_errors(), # return a StringIO with the errors - lp_restored_data=None, - backup_metadata=None, - ) - return asdict(result) - - # Step 2: Extract and validate learning package, entities and collections - # Errors are collected and reported at the end - # No saving to DB happens until all validations pass - learning_package_validated = self._extract_learning_package(organized_files["learning_package"]) - lp_metadata = learning_package_validated.pop("metadata", {}) - - components_validated = self._extract_entities( - organized_files["components"], ComponentSerializer, ComponentVersionSerializer - ) - containers_validated = self._extract_entities( - organized_files["containers"], ContainerSerializer, ContainerVersionSerializer - ) - - collections_validated = self._extract_collections( - organized_files["collections"] - ) - - # Step 3.1: If there are validation errors, return them without saving anything - if self.errors: - result = RestoreResult( - status="error", - log_file_error=self._write_errors(), # return a StringIO with the errors - lp_restored_data=None, - backup_metadata=None, - ) - return asdict(result) - - # Step 3.2: Save everything to the DB - # All validations passed, we can proceed to save everything - # Save the learning package first to get its ID - archive_package_ref = learning_package_validated["package_ref"] - learning_package = self._save( - learning_package_validated, - components_validated, - containers_validated, - collections_validated, - component_static_files=organized_files["component_static_files"] - ) - - num_containers = sum( - len(containers_validated.get(container_type, [])) - for container_type in ["section", "subsection", "unit"] - ) - - org_code, package_code = unpack_package_ref(archive_package_ref) - result = RestoreResult( - status="success", - log_file_error=None, - lp_restored_data=RestoreLearningPackageData( - id=learning_package.id, - package_ref=learning_package.package_ref, - archive_package_ref=archive_package_ref, - archive_org_code=org_code, - archive_package_code=package_code, - title=learning_package.title, - num_containers=num_containers, - num_sections=len(containers_validated.get("section", [])), - num_subsections=len(containers_validated.get("subsection", [])), - num_units=len(containers_validated.get("unit", [])), - num_components=len(components_validated["components"]), - num_collections=len(collections_validated["collections"]), - ), - backup_metadata=BackupMetadata( - format_version=lp_metadata.get("format_version", 1), - created_by=lp_metadata.get("created_by"), - created_by_email=lp_metadata.get("created_by_email"), - created_at=lp_metadata.get("created_at"), - original_server=lp_metadata.get("origin_server"), - ) if lp_metadata else None, - ) - return asdict(result) - - def check_mandatory_files(self) -> Tuple[list[dict[str, Any]], dict[str, Any]]: - """ - Check for the presence of mandatory files in the zip archive. - So far, the only mandatory file is package.toml. - """ - organized_files = self._get_organized_file_list(self.zipf.namelist()) - - if not organized_files["learning_package"]: - self.errors.append({"file": TOML_PACKAGE_NAME, "errors": "Missing learning package file."}) - - return self.errors, organized_files - - # -------------------------- - # Extract + Validate - # -------------------------- - - def _extract_learning_package(self, package_file: str) -> dict[str, Any]: - """Extract and validate the learning package TOML file.""" - toml_content_text = self._read_file_from_zip(package_file) - toml_content_dict = parse_learning_package_toml(toml_content_text) - lp = toml_content_dict.get("learning_package") - lp_metadata = toml_content_dict.get("meta") - - # Validate learning package data - lp_serializer = LearningPackageSerializer(data=lp) - if not lp_serializer.is_valid(): - self.errors.append({"file": f"{package_file} learning package section", "errors": lp_serializer.errors}) - - # Validate metadata if present - lp_metadata_serializer = LearningPackageMetadataSerializer(data=lp_metadata) - if not lp_metadata_serializer.is_valid(): - self.errors.append({"file": f"{package_file} meta section", "errors": lp_metadata_serializer.errors}) - - lp_validated = lp_serializer.validated_data if lp_serializer.is_valid() else {} - lp_metadata = lp_metadata_serializer.validated_data if lp_metadata_serializer.is_valid() else {} - lp_validated["metadata"] = lp_metadata - return lp_validated - - def _extract_entities( - self, - entity_files: list[str], - entity_serializer: type[serializers.Serializer], - version_serializer: type[serializers.Serializer], - ) -> dict[str, Any]: - """Generic extraction + validation pipeline for containers or components.""" - results: dict[str, list[Any]] = defaultdict(list) - - for file in entity_files: - if not file.endswith(".toml"): - # Skip non-TOML files - continue - - entity_data, draft_version, published_version = self._load_entity_data(file) - serializer = entity_serializer( - data={"created": self.utc_now, "created_by": None, **entity_data} - ) - - if not serializer.is_valid(): - self.errors.append({"file": file, "errors": serializer.errors}) - continue - - entity_data = serializer.validated_data - self.all_publishable_entity_refs.add(entity_data["entity_ref"]) - entity_type = entity_data.pop("container_type", "components") - results[entity_type].append(entity_data) - - valid_versions = self._validate_versions( - entity_data, - draft_version, - published_version, - version_serializer, - file=file - ) - if valid_versions["draft"]: - results[f"{entity_type}_drafts"].append(valid_versions["draft"]) - if valid_versions["published"]: - results[f"{entity_type}_published"].append(valid_versions["published"]) - - return results - - def _extract_collections( - self, - collection_files: list[str], - ) -> dict[str, Any]: - """Extraction + validation pipeline for collections.""" - results: dict[str, list[Any]] = defaultdict(list) - - for file in collection_files: - if not file.endswith(".toml"): - # Skip non-TOML files - continue - toml_content = self._read_file_from_zip(file) - collection_data = parse_collection_toml(toml_content) - collection_data = collection_data.get("collection", {}) - serializer = CollectionSerializer(data={"created_by": None, **collection_data}) - if not serializer.is_valid(): - self.errors.append({"file": f"{file} collection section", "errors": serializer.errors}) - continue - collection_validated = serializer.validated_data - entities_list = collection_validated["entities"] - for entity_ref in entities_list: - if entity_ref not in self.all_publishable_entity_refs: - self.errors.append({ - "file": file, - "errors": f"Entity ref {entity_ref} not found for collection {collection_validated.get('key')}" - }) - results["collections"].append(collection_validated) - - return results - - # -------------------------- - # Save Logic - # -------------------------- - - def _save( - self, - learning_package: dict[str, Any], - components: dict[str, Any], - containers: dict[str, Any], - collections: dict[str, Any], - *, - component_static_files: dict[str, List[str]] - ) -> LearningPackage: - """Persist all validated entities in two phases: published then drafts.""" - - # Important: If not using a specific LP ref/key, generate a temporary one - # We cannot use the original key because it may generate security issues - if not self.package_ref: - # Generate a tmp ref for the staged learning package - if not self.user: - raise ValueError("User is required to generate a staged package_ref") - learning_package["package_ref"] = generate_staged_package_ref( - archive_package_ref=learning_package["package_ref"], - user=self.user - ) - else: - learning_package["package_ref"] = self.package_ref - - learning_package_obj = publishing_api.create_learning_package(**learning_package) - self.learning_package_id = learning_package_obj.id - - with publishing_api.bulk_draft_changes_for(learning_package_obj.id): - self._save_components(learning_package_obj, components, component_static_files) - self._save_units(learning_package_obj, containers) - self._save_subsections(learning_package_obj, containers) - self._save_sections(learning_package_obj, containers) - self._save_collections(learning_package_obj, collections) - publishing_api.publish_all_drafts(learning_package_obj.id) - - with publishing_api.bulk_draft_changes_for(learning_package_obj.id): - self._save_draft_versions(components, containers, component_static_files) - - return learning_package_obj - - def _save_collections(self, learning_package, collections): - """Save collections and their entities.""" - for valid_collection in collections.get("collections", []): - entities = valid_collection.pop("entities", []) - collection = collections_api.create_collection( - learning_package.id, created_by=self.user_id, **valid_collection - ) - collection = collections_api.add_to_collection( - learning_package_id=learning_package.id, - collection_code=collection.collection_code, - entities_qset=publishing_api.get_publishable_entities(learning_package.id).filter( - entity_ref__in=entities - ) - ) - - def _save_components(self, learning_package, components, component_static_files): - """Save components and published component versions.""" - for valid_component in components.get("components", []): - entity_ref = valid_component.pop("entity_ref") - component = components_api.create_component(learning_package.id, created_by=self.user_id, **valid_component) - self.components_map_by_ref[entity_ref] = component - - for valid_published in components.get("components_published", []): - entity_ref = valid_published.pop("entity_ref") - version_num = valid_published["version_num"] # Should exist, validated earlier - component = self.components_map_by_ref[entity_ref] - media_to_replace = self._resolve_static_files( - version_num, entity_ref, component.component_type, component_static_files - ) - self.all_published_entities_versions.add( - (entity_ref, version_num) - ) # Track published version - components_api.create_next_component_version( - component.publishable_entity.id, - media_to_replace=media_to_replace, - force_version_num=valid_published.pop("version_num", None), - created_by=self.user_id, - **valid_published - ) - - def _save_container( - self, - learning_package, - containers, - *, - container_cls: containers_api.ContainerSubclass, - container_map: dict, - children_map: dict, - ): - """Internal logic for _save_units, _save_subsections, and _save_sections""" - type_code = container_cls.type_code # e.g. "unit" - for data in containers.get(type_code, []): - entity_ref = data.pop("entity_ref") - container = containers_api.create_container( - learning_package.id, - # As of Verawood, the primary identity of a container is its - # `container_code`. By convention, this equals the `entity_ref` - # (aka `[entity].key`). It's safe to assume that all v1 - # archives have an identical `entity_ref` and `container_code` for each - # entity-container. BUT, this assumpion may not hold true v2+. - container_code=entity_ref, - **data, # should this be allowed to override any of the following fields? - created_by=self.user_id, - container_cls=container_cls, - ) - container_map[entity_ref] = container # e.g. `self.units_map_by_ref[entity_ref] = unit` - - for valid_published in containers.get(f"{type_code}_published", []): - entity_ref = valid_published.pop("entity_ref") - children = self._resolve_children(valid_published, children_map) - version_num = valid_published.pop("version_num", None) - self.all_published_entities_versions.add((entity_ref, version_num)) - containers_api.create_next_container_version( - container_map[entity_ref], - force_version_num=version_num, - **valid_published, # should this be allowed to override any of the following fields? - entities=children, - created_by=self.user_id, - ) - - def _save_units(self, learning_package, containers): - """Save units and published unit versions.""" - self._save_container( - learning_package, - containers, - container_cls=Unit, - container_map=self.units_map_by_ref, - children_map=self.components_map_by_ref, - ) - - def _save_subsections(self, learning_package, containers): - """Save subsections and published subsection versions.""" - self._save_container( - learning_package, - containers, - container_cls=Subsection, - container_map=self.subsections_map_by_ref, - children_map=self.units_map_by_ref, - ) - - def _save_sections(self, learning_package, containers): - """Save sections and published section versions.""" - self._save_container( - learning_package, - containers, - container_cls=Section, - container_map=self.sections_map_by_ref, - children_map=self.subsections_map_by_ref, - ) - - def _save_draft_versions(self, components, containers, component_static_files): - """Save draft versions for all entity types.""" - for valid_draft in components.get("components_drafts", []): - entity_ref = valid_draft.pop("entity_ref") - version_num = valid_draft["version_num"] # Should exist, validated earlier - if self._is_version_already_exists(entity_ref, version_num): - continue - component = self.components_map_by_ref[entity_ref] - media_to_replace = self._resolve_static_files( - version_num, entity_ref, component.component_type, component_static_files - ) - components_api.create_next_component_version( - component.publishable_entity.id, - media_to_replace=media_to_replace, - force_version_num=valid_draft.pop("version_num", None), - # Drafts can diverge from published, so we allow ignoring previous media - # Use case: published v1 had files A, B; draft v2 only has file A - ignore_previous_media=True, - created_by=self.user_id, - **valid_draft - ) - - def _process_draft_containers( - container_cls: containers_api.ContainerSubclass, - container_map: dict, - children_map: dict, - ): - for valid_draft in containers.get(f"{container_cls.type_code}_drafts", []): - entity_ref = valid_draft.pop("entity_ref") - version_num = valid_draft["version_num"] # Should exist, validated earlier - if self._is_version_already_exists(entity_ref, version_num): - continue - children = self._resolve_children(valid_draft, children_map) - del valid_draft["version_num"] - containers_api.create_next_container_version( - container_map[entity_ref], - **valid_draft, # should this be allowed to override any of the following fields? - entities=children, - force_version_num=version_num, - created_by=self.user_id, - ) - - _process_draft_containers(Unit, self.units_map_by_ref, children_map=self.components_map_by_ref) - _process_draft_containers(Subsection, self.subsections_map_by_ref, children_map=self.units_map_by_ref) - _process_draft_containers(Section, self.sections_map_by_ref, children_map=self.subsections_map_by_ref) - - # -------------------------- - # Utilities - # -------------------------- - - def _format_errors(self) -> str: - """Return formatted error content as a string.""" - if not self.errors: - return "" - lines = [f"{err['file']}: {err['errors']}" for err in self.errors] - return "Errors encountered during restore:\n" + "\n".join(lines) + "\n" - - def _write_errors(self) -> StringIO | None: - """ - Write errors to a StringIO buffer. - """ - content = self._format_errors() - if not content: - return None - return StringIO(content) - - def _is_version_already_exists(self, entity_ref: str, version_num: int) -> bool: - """ - Check if a version already exists for a given entity_ref and version number. - - Note: - Skip creating draft if this version is already published - Why? Because the version itself is already created and - we don't want to create duplicate versions. - Otherwise, we will raise an IntegrityError on PublishableEntityVersion - due to unique constraints between publishable_entity and version_num. - """ - identifier = (entity_ref, version_num) - return identifier in self.all_published_entities_versions - - def _resolve_static_files( - self, - num_version: int, - entity_ref: str, - component_type: ComponentType, - static_files_map: dict[str, List[str]] - ) -> dict[str, bytes | int]: - """Resolve static file paths into their binary media content.""" - resolved_files: dict[str, bytes | int] = {} - - static_file_key = f"{entity_ref}:v{num_version}" # e.g., "xblock.v1:html:my_component_123456:v1" - block_type = component_type.name # e.g., "html" - static_files = static_files_map.get(static_file_key, []) - for static_file in static_files: - local_key = static_file.split(f"v{num_version}/")[-1] - with self.zipf.open(static_file, "r") as f: - media_bytes = f.read() - if local_key == "block.xml": - # Special handling for block.xml to ensure - # storing the value as a media instance - if not self.learning_package_id: - raise ValueError("learning_package_id must be set before resolving static files.") - text_media = media_api.get_or_create_text_media( - self.learning_package_id, - media_api.get_or_create_media_type(f"application/vnd.openedx.xblock.v1.{block_type}+xml").id, - text=media_bytes.decode("utf-8"), - created=self.utc_now, - ) - resolved_files[local_key] = text_media.id - else: - resolved_files[local_key] = media_bytes - return resolved_files - - def _resolve_children(self, entity_data: dict[str, Any], lookup_map: dict[str, Any]) -> list[Any]: - """Resolve child entity refs into model instances.""" - children_refs = entity_data.pop("children", []) - return [lookup_map[ref] for ref in children_refs if ref in lookup_map] - - def _load_entity_data( - self, entity_file: str - ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]: - """Load entity data and its versions from TOML.""" - entity_toml_txt = self._read_file_from_zip(entity_file) - entity_toml_dict = parse_publishable_entity_toml(entity_toml_txt) - entity_data = entity_toml_dict.get("entity", {}) - version_data = entity_toml_dict.get("version", []) - return entity_data, *self._get_versions_to_write(version_data, entity_data) - - def _validate_versions(self, entity_data, draft, published, serializer_cls, *, file) -> dict[str, Any]: - """Validate draft/published versions with serializer.""" - valid = {"draft": None, "published": None} - for label, version in [("draft", draft), ("published", published)]: - if not version: - continue - serializer = serializer_cls( - data={ - "entity_ref": entity_data["entity_ref"], - "created": self.utc_now, - "created_by": None, - **version - } - ) - if serializer.is_valid(): - valid[label] = serializer.validated_data - else: - self.errors.append({"file": file, "errors": serializer.errors}) - return valid - - def _read_file_from_zip(self, filename: str) -> str: - """Read and decode a UTF-8 file from the zip archive.""" - with self.zipf.open(filename) as f: - return f.read().decode("utf-8") - - def _get_organized_file_list(self, file_paths: list[str]) -> dict[str, Any]: - """Organize file paths into categories: learning_package, containers, components, collections.""" - organized: dict[str, Any] = { - "learning_package": None, - "containers": [], - "components": [], - "component_static_files": defaultdict(list), - "collections": [], - } - - # This is going to map static file directory roots to the appropriate - # entity refs. - comp_paths_to_refs = {} - - # The ordering of the file processing is important because we need to - # ensure that TOML files for a given component are processed before the - # static files for that component. '.' sorts before '/', so "foo.toml" - # will sort before "foo/component_versions/v1/static/figure1.webp" or - # any other subdirectory of the "foo" component. Processing the TOML - # first allows us to map the directory to a entity ref. - for path in sorted(file_paths): - if path.endswith("/"): - # Skip directories - continue - if path == TOML_PACKAGE_NAME: - organized["learning_package"] = path - elif path.startswith("entities/") and str(Path(path).parent) == "entities" and path.endswith(".toml"): - # Top-level entity TOML files are considered containers - organized["containers"].append(path) - elif path.startswith("entities/"): - if path.endswith(".toml"): - # Component entity TOML files - organized["components"].append(path) - component_toml_str = self._read_file_from_zip(path) - component_toml = tomllib.loads(component_toml_str) - entity_ref = component_toml['entity']['key'] - comp_path = path.removesuffix(".toml") - - # This maps the root path of a component, e.g."entities/xblock.v1/html/my_component_a822bb" - # to the actual ref, e.g. "xblock.v1:html:my_component". The last part of the ref will - # often correlate to the directory name, but does not have to (a hash is sometimes added). - comp_paths_to_refs[comp_path] = entity_ref - - else: - # Component static files - # Path structure: entities////component_versions//static/... - # Example: entities/xblock.v1/html/my_component_a822bb/component_versions/v1/static/... - - # e.g. 'entities/xblock.v1/html/my_component_a822bb' - component_root_path = '/'.join(Path(path).parts[0:4]) - - try: - component_ref = comp_paths_to_refs[component_root_path] - except KeyError: - self.errors.append( - { - "file": path, - "errors": f"Missing component TOML file at {component_root_path}.toml" - } - ) - continue - - num_version = Path(path).parts[5] if len(Path(path).parts) > 5 else "v1" # e.g., 'v1' - - component_ref += f":{num_version}" - organized["component_static_files"][component_ref].append(path) - - elif path.startswith("collections/") and path.endswith(".toml"): - # Collection TOML files - organized["collections"].append(path) - - return organized - - def _get_versions_to_write( - self, - version_data: list[dict[str, Any]], - entity_data: dict[str, Any] - ) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]: - """Return the draft and published versions to write, based on entity data.""" - draft_num = entity_data.get("draft", {}).get("version_num") - published_num = entity_data.get("published", {}).get("version_num") - lookup = {v.get("version_num"): v for v in version_data} - return ( - lookup.get(draft_num) if draft_num else None, - lookup.get(published_num) if published_num else None, - ) diff --git a/src/openedx_content/management/commands/lp_load.py b/src/openedx_content/management/commands/lp_load.py index 97fc55870..2c5eef075 100644 --- a/src/openedx_content/management/commands/lp_load.py +++ b/src/openedx_content/management/commands/lp_load.py @@ -8,7 +8,8 @@ from django.core.management import CommandError from django.core.management.base import BaseCommand -from openedx_content.applets.backup_restore.api import load_learning_package +from openedx_content.applets.backup_restore.api import load_learning_package_from_path +from openedx_content.applets.backup_restore.errors import BackupRestoreError, RestoreFailedError logger = logging.getLogger(__name__) @@ -17,41 +18,52 @@ class Command(BaseCommand): """ - Django management command to load a learning package from a zip file. + Django management command to load a learning package from a backup archive. """ - help = 'Load a learning package from a zip file.' + help = 'Load a learning package from a backup archive (a .zip file or an unzipped directory).' def add_arguments(self, parser): - parser.add_argument('file_name', type=str, help='The path of the input zip file to load.') + parser.add_argument( + 'path', + type=str, + help='The path of the archive to load: either a .zip file or a directory.', + ) parser.add_argument('username', type=str, help='The username of the user performing the load operation.') + parser.add_argument( + '--package-ref', + type=str, + default=None, + help=( + "The package ref to restore under. If omitted, a staged ref " + "namespaced to the user is generated." + ), + ) def handle(self, *args, **options): - file_name = options['file_name'] + path = options['path'] username = options['username'] - if not file_name.lower().endswith(".zip"): - raise CommandError("Input file name must end with .zip") + package_ref = options['package_ref'] + try: - start_time = time.time() - # Get the user performing the operation user = User.objects.get(username=username) + except User.DoesNotExist as exc: + raise CommandError(f"No such user: {username}") from exc + + start_time = time.time() + try: + result = load_learning_package_from_path(path, user=user, package_ref=package_ref) + except RestoreFailedError as exc: + # The archive is bad. Show every problem we found, not just the first. + raise CommandError(exc.as_text()) from exc + except BackupRestoreError as exc: + raise CommandError(f"Failed to load '{path}': {exc}") from exc + except Exception as exc: + logger.exception("Failed to load archive %s", path) + raise CommandError(f"Failed to load '{path}': {exc}") from exc - result = load_learning_package(file_name, user=user) - duration = time.time() - start_time - if result["status"] == "error": - message = "Errors encountered during restore:\n" - log_buffer = result.get("log_file_error") - if log_buffer: - message += log_buffer.getvalue() - raise CommandError(message) - message = f'{file_name} loaded successfully (duration: {duration:.2f} seconds)' - self.stdout.write(self.style.SUCCESS(message)) - except FileNotFoundError as exc: - message = f"Learning package file {file_name} not found: {exc}" - raise CommandError(message) from exc - except Exception as e: - message = f"Failed to load '{file_name}': {e}" - logger.exception( - "Failed to load zip file %s ", - file_name, - ) - raise CommandError(message) from e + duration = time.time() - start_time + restored = result.lp_restored_data + self.stdout.write(self.style.SUCCESS( + f'{path} loaded successfully as "{restored.package_ref}" ' + f'(duration: {duration:.2f} seconds)' + )) diff --git a/test_utils/zip_file_utils.py b/test_utils/zip_file_utils.py index 31dc4c589..c9876abcb 100644 --- a/test_utils/zip_file_utils.py +++ b/test_utils/zip_file_utils.py @@ -24,3 +24,42 @@ def folder_to_inmemory_zip(folder_path: str) -> zipfile.ZipFile: zipf.write(file_path, arcname=str(arcname)) buffer.seek(0) return zipfile.ZipFile(buffer, "r") + + +def folder_to_zip_path( + folder_path: str, + dest_dir: str, + name: str = "archive.zip", + prefix: str = "", + extra_names: tuple = (), +) -> str: + """ + Write the contents of a folder out as a real zip file on disk. + + Unlike ``folder_to_inmemory_zip``, this returns a *path*, which is what the + restore pipeline takes (it opens the archive itself, so that it can support + both zip files and plain directories). + + Args: + folder_path (str): Path to the folder to zip. + dest_dir (str): Directory to write the zip file into. + name (str): File name to give the zip file. + prefix (str): Prepended to every archive member, e.g. ``"MyLib/"``. Use + this to build the kind of archive you get from ``zip -r x.zip MyLib``, + where everything sits inside a single wrapper folder. + extra_names (tuple): Extra (empty) members to add, for simulating the + debris real archiving tools leave behind, e.g. ``"__MACOSX/._MyLib"``. + + Returns: + str: The path of the zip file that was written. + """ + folder = Path(folder_path) + zip_path = Path(dest_dir) / name + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + for file_path in sorted(folder.rglob("*")): + if file_path.is_file(): + arcname = prefix + str(file_path.relative_to(folder)) + zipf.write(file_path, arcname=arcname) + for extra_name in extra_names: + zipf.writestr(extra_name, b"") + return str(zip_path) diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml new file mode 100644 index 000000000..67863ec25 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml @@ -0,0 +1,15 @@ +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml new file mode 100644 index 000000000..abee428a5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml @@ -0,0 +1,17 @@ +# Redeclares the same entity key as first.toml, which would silently overwrite +# it if we didn't check. +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1 Again" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep b/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep new file mode 100644 index 000000000..1c3071fb4 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep @@ -0,0 +1 @@ +# Intentionally empty: an archive with no package.toml at all. diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml new file mode 100644 index 000000000..f284a5753 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml @@ -0,0 +1,12 @@ +# The [learning_package] table is missing its mandatory "key" field. +[meta] +format_version = 1 +created_by = "lp_user" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml new file mode 100644 index 000000000..07d64b11c --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml @@ -0,0 +1,7 @@ +# The mandatory [meta] table is missing entirely. +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml new file mode 100644 index 000000000..993a94d82 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml @@ -0,0 +1,16 @@ +# "chapter" is not a container type this version of the code can build. +[entity] +can_stand_alone = true +key = "chapter1-abc123" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.chapter] + +[[version]] +title = "Chapter 1" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml new file mode 100644 index 000000000..4e53fd2f0 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml @@ -0,0 +1,19 @@ +# This unit lists a child that isn't defined anywhere in the archive. +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1" +version_num = 1 + +[version.container] +children = ["xblock.v1:html:does-not-exist"] diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml new file mode 100644 index 000000000..4a2c10b43 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml @@ -0,0 +1,11 @@ +# We only know how to read format_version 1.x. +[meta] +format_version = 2 +created_by = "lp_user" +created_at = 2025-10-05T18:23:45.180535Z + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml new file mode 100644 index 000000000..9abcf1046 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[collection] +title = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml new file mode 100644 index 000000000..5ff8122fe --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml @@ -0,0 +1,15 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 +created_by = "eddy" +created_by_email = "eddy@axim.org" +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml new file mode 100644 index 000000000..9abcf1046 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[collection] +title = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml new file mode 100644 index 000000000..09efb6975 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml @@ -0,0 +1,6 @@ +[collection] +title = "Practice Exams" +key = "dupe-collection-key" +description = "" +created = 2026-03-11T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml new file mode 100644 index 000000000..024fc2277 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml @@ -0,0 +1,8 @@ +# Same key as dupe_1.toml, which is not allowed -- Collection keys must be +# unique within a Learning Package. +[collection] +title = "Difficult Problems" +key = "dupe-collection-key" +description = "" +created = 2026-03-12T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml new file mode 100644 index 000000000..586a2439a --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml @@ -0,0 +1,8 @@ +# The problem here is that title and key are not in a table, and we don't allow +# that. +title = "Difficult Problems" +key = "difficult-problems" + +[collection] +description = "" +created = 2026-03-11T19:20:20.394360Z diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml new file mode 100644 index 000000000..9d5a89d94 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml @@ -0,0 +1,7 @@ +# Typo: [collektion] instead of [collection] +[collektion] +title = "Difficult Problems" +key = "difficult-problems" +description = "" +created = 2026-03-11T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml new file mode 100644 index 000000000..c2ccd0582 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml @@ -0,0 +1,9 @@ +[collection] +title = "Difficult Problems" +key = "difficult-problems" +description = "The tricky ones. 🐢" +created = 2026-03-11T19:20:20.394360Z +entities = [ + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000001", + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000002", +] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep b/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep new file mode 100644 index 000000000..847e98cad --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep @@ -0,0 +1,2 @@ +# Intentionally empty: this directory exists so that fsspec has something +# to point at for the "archive with nothing in it" test case. diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml new file mode 100644 index 000000000..6537aa087 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml @@ -0,0 +1,2 @@ +[entity] +key = " diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml new file mode 100644 index 000000000..ffee075be --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml @@ -0,0 +1,10 @@ +# No [entity] key +[entity] +can_stand_alone = true +created = 2025-10-31T16:42:04.158245Z + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml new file mode 100644 index 000000000..9451c4173 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml @@ -0,0 +1,11 @@ +# Typo: [entitty] instead of [entity] +[entitty] +can_stand_alone = true +key = "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" +created = 2025-10-31T16:42:04.158245Z + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml new file mode 100644 index 000000000..5448ff34d --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml @@ -0,0 +1,13 @@ +# An entity with no [[version]] tables at all. Extraction should still succeed +# and give us an empty "versions" list -- it's the validation step's job to +# decide whether that's acceptable. +[entity] +can_stand_alone = true +key = "no-versions-c0ffee" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] + +[entity.published] + +[entity.container.unit] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml new file mode 100644 index 000000000..d86a9d4ad --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml @@ -0,0 +1,20 @@ +[entity] +can_stand_alone = true +key = "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 3 + +[entity.published] +version_num = 2 + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 + +[[version]] +title = "Text" +version_num = 2 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml new file mode 100644 index 000000000..e4fe6d5af --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml @@ -0,0 +1 @@ +

Version 2 text.

diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml new file mode 100644 index 000000000..50edb9d16 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml @@ -0,0 +1 @@ +

Version 3 text.

diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png new file mode 100644 index 000000000..9d17883bc --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png @@ -0,0 +1 @@ +PNG-ish bytes for testing diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml new file mode 100644 index 000000000..39ff085ff --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml @@ -0,0 +1,29 @@ +# Pretty normal Section with a couple of child Subsections +[entity] +can_stand_alone = true +key = "section-9-ac4b9f" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 2 + +[entity.published] +version_num = 1 + +[entity.container.section] + +# ### Versions + +[[version]] +title = "Section 9" +version_num = 2 + +[version.container] +children = ["week-7-e73782", "subsection-001-e4bbe5"] + +[[version]] +title = "Section 9" +version_num = 1 + +[version.container] +children = ["week-7-e73782"] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml new file mode 100644 index 000000000..ab7a14cac --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml @@ -0,0 +1,26 @@ +# Neither [entity.future_thing] nor [future_top_level] is something this version +# of the code knows about. The nested one is kept, so that a newer export can add +# entity attributes without older code choking on them (and so we can warn about +# probable typos). The top-level one is outside the entity, so it isn't part of +# the entity data we assemble. +[entity] +can_stand_alone = true +key = "unknown-table-abc123" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.future_thing] +some_setting = "hello" + +[future_top_level] +irrelevant = true + +# ### Versions + +[[version]] +title = "Some Entity" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml new file mode 100644 index 000000000..09db43a4a --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[meta] +format_version = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml new file mode 100644 index 000000000..2b05c95ab --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml @@ -0,0 +1,15 @@ +# The problem here is that created_by and formt_version are not in a table, and +# we don't allow that. +created_by = "eddy" +format_version = 1 + +[meta] +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml new file mode 100644 index 000000000..4874bb574 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml @@ -0,0 +1,9 @@ +# This is the absolute minimum necessary to pass the payload extraction step. +# It doesn't matter that required fields are missing --> that's handled in +# validation. Payload extraction just requires that the tables we expect exist. +# +# Ordering is also irrelevant for this file. +[learning_package] + +[meta] +format_version = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml new file mode 100644 index 000000000..03e087cf7 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml @@ -0,0 +1,3 @@ +# This errors because the [learning_package] section is missing. +[meta] +format_version = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml new file mode 100644 index 000000000..95c65a9dd --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml @@ -0,0 +1,4 @@ +# This errors because the [meta] section is missing. + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml new file mode 100644 index 000000000..5ff8122fe --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml @@ -0,0 +1,15 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 +created_by = "eddy" +created_by_email = "eddy@axim.org" +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml new file mode 100644 index 000000000..36d1161ea --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml @@ -0,0 +1,10 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 + +[learning_package] +title = "Fun Library" + +[unknown] +new_field = "Aloha!" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml new file mode 100644 index 000000000..bbb1d49e2 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml @@ -0,0 +1,7 @@ +# We're going to accept format_version 1.1 as a hedge against backwards +# compatible additions to this format (we may never use this). +[meta] +format_version = 1.1 + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml new file mode 100644 index 000000000..dc2227b41 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml @@ -0,0 +1,6 @@ +# We don't support format_version > 1 +[meta] +format_version = 2 + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml new file mode 100644 index 000000000..8a9df5dc2 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml @@ -0,0 +1,6 @@ +# format_version needs to be an integer +[meta] +format_version = "b" + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml new file mode 100644 index 000000000..d23b42927 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml @@ -0,0 +1,8 @@ +# `true` is not a version number. This matters because Python's bool is a +# subclass of int, so a naive numeric check would read this as version 1. +[meta] +format_version = true + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" diff --git a/tests/openedx_content/applets/backup_restore/test_archive.py b/tests/openedx_content/applets/backup_restore/test_archive.py new file mode 100644 index 000000000..89befdf6e --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_archive.py @@ -0,0 +1,69 @@ +""" +Tests for resolving an archive location into a filesystem we can read. + +These tests are strictly for the archive module, and therefore don't need Django +to run. +""" +import tempfile +import zipfile +from pathlib import Path +from unittest import TestCase + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem + +from openedx_content.applets.backup_restore import archive +from openedx_content.applets.backup_restore.errors import ArchiveNotReadableError + + +class ReadFsForPathTest(TestCase): + """Tests for resolving a path into a readable filesystem.""" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = Path(self._tmp.name) + + def _make_zip(self, name: str) -> Path: + zip_path = self.tmp_path / name + with zipfile.ZipFile(zip_path, "w") as zipf: + zipf.writestr("package.toml", "[meta]\nformat_version = 1\n") + return zip_path + + def test_directory(self): + contents_dir = self.tmp_path / "unzipped" + contents_dir.mkdir() + (contents_dir / "package.toml").write_text("[meta]\nformat_version = 1\n") + + fs = archive.read_fs_for_path(str(contents_dir)) + + assert isinstance(fs, DirFileSystem) + assert fs.exists("package.toml") + + def test_zip_file(self): + fs = archive.read_fs_for_path(str(self._make_zip("backup.zip"))) + + assert isinstance(fs, ZipFileSystem) + assert fs.exists("package.toml") + + def test_zip_file_uppercase_suffix(self): + """Suffix matching is case-insensitive.""" + fs = archive.read_fs_for_path(str(self._make_zip("BACKUP.ZIP"))) + + assert isinstance(fs, ZipFileSystem) + + def test_nonexistent_path(self): + missing = str(self.tmp_path / "not_here.zip") + + with self.assertRaises(ArchiveNotReadableError) as ctx: + archive.read_fs_for_path(missing) + assert ctx.exception.path == missing + + def test_file_that_is_not_a_zip(self): + not_a_zip = self.tmp_path / "package.toml" + not_a_zip.write_text("[meta]\nformat_version = 1\n") + + with self.assertRaises(ArchiveNotReadableError) as ctx: + archive.read_fs_for_path(str(not_a_zip)) + assert ctx.exception.path == str(not_a_zip) diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py new file mode 100644 index 000000000..8fc89deb2 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -0,0 +1,723 @@ +""" +Tests for restoring a Learning Package from a backup archive. + +These drive the public API (``backup_restore.api``) rather than the loader +directly, so that the whole pipeline -- archive, payload, validation, loading -- +is exercised the way a caller would use it. + +The ``library_backup`` fixture is the workhorse here. It is a real archive +produced by the backup side, and several of its quirks are deliberate: + +* ``unit1`` has a blank version title, because untitled units are common in + content imported from courses. +* ``section1-extra-8ca126.toml`` and ``...-extra.toml`` have filenames that don't + match the entity key inside them, because the backup side hashes filenames to + avoid collisions. +* Its entities cover every draft/published combination we care about. +""" +import os +import shutil +import tempfile +from datetime import datetime, timezone +from io import StringIO + +from django.contrib.auth import get_user_model +from django.core.management import CommandError, call_command +from django.test import TestCase + +from openedx_content.applets.backup_restore import api +from openedx_content.applets.backup_restore.errors import ( + ArchiveNotReadableError, + MissingFileError, + RestoreFailedError, + SchemaError, + TableNotFoundError, + UnknownContainerTypeError, + UnresolvedChildError, + UnsupportedFormatError, +) +from openedx_content.applets.backup_restore.loading import Loader +from openedx_content.applets.backup_restore.results import generate_staged_package_ref +from openedx_content.applets.backup_restore.schema import CompletePackageInputData, EntityInputData +from openedx_content.applets.backup_restore.validation import ValidatedLearningPackageInput +from openedx_content.applets.collections import api as collections_api +from openedx_content.applets.components import api as components_api +from openedx_content.applets.containers import api as containers_api +from openedx_content.applets.publishing import api as publishing_api +from test_utils.zip_file_utils import folder_to_zip_path + +User = get_user_model() + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") +LIBRARY_BACKUP_DIR = os.path.join(FIXTURES_DIR, "library_backup") + + +def broken_fixture(name: str) -> str: + return os.path.join(FIXTURES_DIR, "broken", name) + + +class RestoreTestCase(TestCase): + """Base test case for restore tests.""" + + def setUp(self): + super().setUp() + self.fixtures_folder = LIBRARY_BACKUP_DIR + self.package_ref = "lib:WGU:LIB_C001" + self.user = User.objects.create_user(username="lp_user", password="12345") + + def as_zip(self, folder: str) -> str: + """Write a fixture folder out as a real zip file and return its path.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + return folder_to_zip_path(folder, tmp_dir.name) + + +class RestoreLearningPackageTest(RestoreTestCase): + """Restoring a well-formed archive.""" + + def test_restore_with_explicit_package_ref(self): + result = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + assert result.status == "success" + assert result.log_file_error is None + + restored = result.lp_restored_data + assert restored.package_ref == "lib-xx:WGU:LIB_C001" + assert restored.archive_package_ref == "lib:WGU:LIB_C001" + assert restored.archive_org_code == "WGU" + assert restored.archive_package_code == "LIB_C001" + assert restored.title == "Library test" + assert restored.num_containers == 3 + assert restored.num_sections == 1 + assert restored.num_subsections == 1 + assert restored.num_units == 1 + assert restored.num_components == 7 + assert restored.num_collections == 1 + + lp = publishing_api.LearningPackage.objects.filter( + package_ref="lib-xx:WGU:LIB_C001" + ).first() + assert lp is not None, "Learning package was not restored." + + def test_learning_package_fields_come_from_the_archive(self): + result = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + lp = publishing_api.LearningPackage.objects.get(id=result.lp_restored_data.id) + assert lp.title == "Library test" + assert lp.description == "" + assert lp.created == datetime(2025, 8, 19, 4, 25, 10, 988166, tzinfo=timezone.utc) + + def test_restore_with_staged_package_ref(self): + """Without an explicit ref we generate one namespaced to the user.""" + result = api.load_learning_package_from_path(self.fixtures_folder, user=self.user) + + assert result.status == "success" + restored_ref = result.lp_restored_data.package_ref + assert result.lp_restored_data.archive_package_ref == "lib:WGU:LIB_C001" + assert restored_ref.startswith("lp-restore:lp_user:WGU:LIB_C001:") + + lp = publishing_api.LearningPackage.objects.filter( + package_ref=restored_ref + ).first() + assert lp is not None, "Learning package with staged ref was not restored." + + def test_backup_metadata(self): + result = api.load_learning_package_from_path(self.fixtures_folder, user=self.user) + + assert result.status == "success" + assert result.backup_metadata.format_version == 1 + assert result.backup_metadata.created_by == "lp_user" + assert result.backup_metadata.created_by_email == "lp_user@example.com" + assert result.backup_metadata.created_at == datetime( + 2025, 10, 5, 18, 23, 45, 180535, tzinfo=timezone.utc + ) + assert result.backup_metadata.original_server == "cms.test" + + def test_restore_from_zip_matches_restore_from_directory(self): + """ + A zip archive and the same archive unzipped must load identically. + + Reading directly from a directory is new -- the old implementation only + accepted zip files -- so it's worth pinning that the two agree. + """ + from_dir = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib:from:dir" + ) + from_zip = api.load_learning_package_from_path( + self.as_zip(self.fixtures_folder), user=self.user, package_ref="lib:from:zip" + ) + + def comparable(result): + data = dict(vars(result.lp_restored_data)) + # These differ by construction. + del data["id"] + del data["package_ref"] + return data + + assert comparable(from_dir) == comparable(from_zip) + assert vars(from_dir.backup_metadata) == vars(from_zip.backup_metadata) + + def test_blank_container_title(self): + """ + Restoring should succeed when a container version has a blank title. + + Blank titles are legal and common -- content imported from courses + (e.g. via the modulestore migrator) frequently has untitled units, and + such content can be backed up. Restoring that same archive must work. + + The ``library_backup`` fixture's ``unit1`` deliberately has a blank + title to exercise this path. + """ + result = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + assert result.status == "success" + lp = publishing_api.LearningPackage.objects.get(id=result.lp_restored_data.id) + unit = containers_api.get_containers(learning_package_id=lp.id).get( + publishable_entity__entity_ref="unit1-b7eafb" + ) + draft_version = publishing_api.get_draft_version(unit.publishable_entity.id) + assert draft_version.title == "" + + def test_entity_key_need_not_match_filename(self): + """ + The key inside the file wins, not the filename. + + The backup side hashes filenames to keep them unique and filesystem-safe, + so the two routinely differ. Two fixture files are named to make sure we + don't accidentally start trusting the filename. + """ + result = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + lp_id = result.lp_restored_data.id + + entity_refs = set( + publishing_api.get_publishable_entities(lp_id).values_list( + "entity_ref", flat=True + ) + ) + assert "section1-8ca126" in entity_refs + assert "section1-extra-8ca126" not in entity_refs + assert "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" in entity_refs + + +class RestoreContentTest(RestoreTestCase): + """Verifies what actually landed in the database.""" + + def setUp(self): + super().setUp() + result = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + self.lp = publishing_api.LearningPackage.objects.get( + id=result.lp_restored_data.id + ) + + def draft_and_published(self, entity_id): + return ( + publishing_api.get_draft_version(entity_id), + publishing_api.get_published_version(entity_id), + ) + + def test_containers(self): + """Verify the containers and their versions were restored correctly.""" + container_qs = containers_api.get_containers(learning_package_id=self.lp.id) + expected = { + # entity_ref: (type, draft version, published version) + "unit1-b7eafb": ("unit", 2, 2), + "subsection1-48afa3": ("subsection", 2, None), + "section1-8ca126": ("section", 2, None), + } + assert {c.entity_ref for c in container_qs} == set(expected) + + for container in container_qs: + container_type, draft_num, published_num = expected[container.entity_ref] + assert containers_api.get_container_type_code_of(container) == container_type + assert container.created_by is not None + assert container.created_by.username == "lp_user" + + draft, published = self.draft_and_published(container.publishable_entity.id) + assert draft is not None + assert draft.version_num == draft_num + assert draft.created_by.username == "lp_user" + if published_num is None: + assert published is None + else: + assert published is not None + assert published.version_num == published_num + assert published.created_by.username == "lp_user" + + def test_components(self): + """ + Verify the components and their versions were restored correctly. + + The version numbers here are the interesting part: they cover draft == + published, draft ahead of published, and never-published. + """ + expected = { + # entity_ref: (component type, draft version, published version) + "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27": + ("drag-and-drop-v2", 2, None), + "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37": ("html", 5, 4), + "xblock.v1:openassessment:1ee38208-a585-4455-a27e-4930aa541f53": + ("openassessment", 2, None), + "xblock.v1:problem:256739e8-c2df-4ced-bd10-8156f6cfa90b": ("problem", 2, None), + "xblock.v1:survey:6681da3f-b056-4c6e-a8f9-040967907471": ("survey", 1, None), + "xblock.v1:video:22601ebd-9da8-430b-9778-cfe059a98568": ("video", 3, None), + "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2": ("html", 2, 2), + } + component_qs = components_api.get_components(self.lp.id) + assert {c.entity_ref for c in component_qs} == set(expected) + + for component in component_qs: + type_name, draft_num, published_num = expected[component.entity_ref] + assert component.component_type.name == type_name + assert component.component_type.namespace == "xblock.v1" + assert component.created_by is not None + assert component.created_by.username == "lp_user" + + draft, published = self.draft_and_published(component.publishable_entity.id) + assert draft is not None + assert draft.version_num == draft_num + assert draft.created_by.username == "lp_user" + if published_num is None: + assert published is None + else: + assert published is not None + assert published.version_num == published_num + assert published.created_by.username == "lp_user" + + def test_block_xml_becomes_text_media(self): + component = components_api.get_components(self.lp.id).get( + publishable_entity__entity_ref=( + "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27" + ) + ) + draft = publishing_api.get_draft_version(component.publishable_entity.id) + + media = draft.componentversion.media.all() + assert media.count() == 1 + block_xml = media.first() + assert " str: + """Copy a fixture folder one level down inside a fresh temp directory.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + shutil.copytree(folder, os.path.join(tmp_dir.name, wrapper)) + return tmp_dir.name + + def comparable(self, result): + """Everything about a restore except what differs by construction.""" + data = dict(vars(result.lp_restored_data)) + del data["id"] + del data["package_ref"] + return data + + def test_wrapper_zip_matches_flat_archive(self): + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + wrapped_zip = folder_to_zip_path( + self.fixtures_folder, tmp_dir.name, prefix="MyLib/" + ) + + flat = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib:flat:one" + ) + wrapped = api.load_learning_package_from_path( + wrapped_zip, user=self.user, package_ref="lib:wrapped:zip" + ) + + assert self.comparable(wrapped) == self.comparable(flat) + assert vars(wrapped.backup_metadata) == vars(flat.backup_metadata) + + def test_wrapper_directory_matches_flat_archive(self): + flat = api.load_learning_package_from_path( + self.fixtures_folder, user=self.user, package_ref="lib:flat:two" + ) + wrapped = api.load_learning_package_from_path( + self.as_wrapper_dir(self.fixtures_folder), + user=self.user, + package_ref="lib:wrapped:dir", + ) + + assert self.comparable(wrapped) == self.comparable(flat) + + def test_macos_style_zip(self): + """A zip made with Finder's "Compress" carries __MACOSX and .DS_Store.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + mac_zip = folder_to_zip_path( + self.fixtures_folder, + tmp_dir.name, + prefix="MyLib/", + extra_names=("__MACOSX/._MyLib", ".DS_Store"), + ) + + result = api.load_learning_package_from_path( + mac_zip, user=self.user, package_ref="lib:wrapped:mac" + ) + + assert result.status == "success" + assert result.lp_restored_data.num_components == 7 + + def test_static_assets_survive_a_wrapper(self): + """ + Static files still resolve when the archive is wrapped. + + The file pointers written during extraction are relative to the + re-rooted filesystem, so this is what catches the case where the wrong + filesystem gets handed downstream -- it would break images for wrapper + archives only. + """ + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + wrapped_zip = folder_to_zip_path( + self.fixtures_folder, tmp_dir.name, prefix="MyLib/" + ) + + result = api.load_learning_package_from_path( + wrapped_zip, user=self.user, package_ref="lib:wrapped:static" + ) + + component = components_api.get_components(result.lp_restored_data.id).get( + publishable_entity__entity_ref=( + "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" + ) + ) + draft = publishing_api.get_draft_version(component.publishable_entity.id) + by_path = { + cvm.path: cvm.media + for cvm in draft.componentversion.componentversionmedia_set.all() + } + assert set(by_path) == {"block.xml", "static/me.png"} + assert by_path["static/me.png"].has_file + assert by_path["static/me.png"].read_file().read() + + def test_ambiguous_archive_is_refused(self): + """ + Several candidate folders means we don't guess. + + ``fixtures/broken/`` holds a handful of directories that each contain a + package.toml, so there is no single obvious root. + """ + packages_before = publishing_api.LearningPackage.objects.count() + + with self.assertRaises(RestoreFailedError) as ctx: + api.load_learning_package_from_path( + os.path.join(FIXTURES_DIR, "broken"), user=self.user + ) + + assert publishing_api.LearningPackage.objects.count() == packages_before + assert any( + isinstance(err, MissingFileError) for err in ctx.exception.errors + ) + + def test_detected_root_is_reported_in_the_error_text(self): + """ + When we do re-root, say so -- every path we report is relative to it. + """ + wrapper_dir = self.as_wrapper_dir(broken_fixture("missing_lp_key")) + + with self.assertRaises(RestoreFailedError) as ctx: + api.load_learning_package_from_path(wrapper_dir, user=self.user) + + text = ctx.exception.as_text() + assert "Archive root: MyLib/" in text + assert "package.toml: learning_package.key" in text diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py new file mode 100644 index 000000000..d3821ee6c --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -0,0 +1,681 @@ +""" +IMPORTANT: If you are adding new fields/behaviors, they should take the form of +*new* tests on new test data files, and not modifications to existing ones. +Please be very cautious about whether you are breaking backwards compatibility. + +This module tests our ability to extract data from the backup archive TOML files +and resources, and assemble them into a combined document that represents the +entire LearningPackage, and is encapsulated in UnvalidatedLearningPackageInput. +Most of these test methods examine individual files. PayloadExtractor takes the +filesystem once, at construction, and its methods take a path, so it should be +possible to do simple test calls on TOML files and dirs without having to mock +anything. + +These tests are strictly for the payload module, and therefore don't need Django +to run. +""" + +import io +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from unittest import TestCase + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem + +from openedx_content.applets.backup_restore import payload + +TEST_DATA_ROOT = Path(__file__).parent / "payload_test_data" +FIXTURES_ROOT = Path(__file__).parent / "fixtures" + + +class ExtractRootPackageFileTest(TestCase): + """Tests for reading package.toml.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "root_packages") + cls.extractor = payload.PayloadExtractor(cls.fs) + + @classmethod + def tearDownClass(cls): + del cls.fs + del cls.extractor + super().tearDownClass() + + def test_file_not_found(self): + with self.assertRaises(payload.MissingFileError) as ctx: + self.extractor.extract_root_package_data("does_not_exist.toml") + assert ctx.exception.path == "does_not_exist.toml" + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as ctx: + self.extractor.extract_root_package_data("broken.toml") + assert ctx.exception.path == "broken.toml" + + def test_fields_not_in_table(self): + with self.assertRaises(payload.FieldsNotInTable) as ctx: + self.extractor.extract_root_package_data("fields_not_in_table.toml") + assert ctx.exception.path == "fields_not_in_table.toml" + assert ctx.exception.fields == ["created_by", "format_version"] + + def test_missing_meta_table(self): + with self.assertRaises(payload.TableNotFoundError) as ctx: + self.extractor.extract_root_package_data("missing_meta.toml") + assert ctx.exception.path == "missing_meta.toml" + assert ctx.exception.table == "meta" + assert "[meta]" in str(ctx.exception) + + def test_missing_learning_package_table(self): + with self.assertRaises(payload.TableNotFoundError) as ctx: + self.extractor.extract_root_package_data("missing_learning_package.toml") + assert ctx.exception.path == "missing_learning_package.toml" + assert ctx.exception.table == "learning_package" + assert "[learning_package]" in str(ctx.exception) + + def test_unsupported_format_version(self): + # We don't support format_version=2 + with self.assertRaises(payload.UnsupportedFormatError): + self.extractor.extract_root_package_data("unsupported_format_version_2.toml") + # We don't support format_version as anthing other than number + with self.assertRaises(payload.UnsupportedFormatError): + self.extractor.extract_root_package_data("unsupported_format_version_b.toml") + + # ...and a boolean is not a number, even though Python's bool is a + # subclass of int and would otherwise read as version 1. + with self.assertRaises(payload.UnsupportedFormatError): + self.extractor.extract_root_package_data("unsupported_format_version_true.toml") + + # We will allow format_version 1.x though, in case we want to extend our + # format in a fully backwards compatible way. + root_data = self.extractor.extract_root_package_data("unsupported_format_version_1_1.toml") + assert root_data["meta"]["format_version"] == 1.1 + + def test_ignore_unknown_tables(self): + """Allow for forwards compatibility.""" + assert "unknown" in self.extractor.extract_root_package_data("unknown_table.toml") + + def test_minimal(self): + data = self.extractor.extract_root_package_data("minimal.toml") + assert data == { + "meta": { + "format_version": 1, + }, + "learning_package": {}, + } + + def test_normal(self): + data = self.extractor.extract_root_package_data("normal_ulmo_v1.toml") + assert data == { + "meta": { + "format_version": 1, + "created_by": "eddy", + "created_by_email": "eddy@axim.org", + "created_at": datetime( + 2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc + ), + "origin_server": "studio.local.openedx.io", + }, + "learning_package": { + "title": "Fun Library", + "key": "lib:Axim:FunLib", + "description": "My very fun library! 🐢", + "created": datetime( + 2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc + ), + "updated": datetime( + 2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc + ), + }, + } + + +class ExtractEntityDataTest(TestCase): + """Tests for reading a single entity TOML file.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "entities") + cls.extractor = payload.PayloadExtractor(cls.fs) + + @classmethod + def tearDownClass(cls): + del cls.fs + del cls.extractor + super().tearDownClass() + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as ctx: + self.extractor.extract_entity_data("broken.toml") + assert ctx.exception.path == "broken.toml" + + def test_missing_entity_table(self): + with self.assertRaises(payload.TableNotFoundError) as ctx: + self.extractor.extract_entity_data("missing_entity_table.toml") + assert ctx.exception.path == "missing_entity_table.toml" + assert ctx.exception.table == "entity" + assert "[entity]" in str(ctx.exception) + + def test_missing_entity_key(self): + """ + An entity with no key extracts fine; rejecting it is validation's job. + + See test_validation.DuplicateEntityTest.test_missing_entity_key. + """ + data = self.extractor.extract_entity_data("missing_entity_key.toml") + assert "key" not in data + assert data["src_path"] == "missing_entity_key.toml" + + def test_dupes(self): + """ + Two files declaring the same entity key both survive extraction. + + This is the property that lets validation reject the duplicate and name + both files. If we collapsed entities into a dict keyed by their ref, the + second definition would overwrite the first and there would be nothing + left to report. + """ + paths = ["dupe_1.toml", "dupe_2.toml"] + data, errors = self.extractor.extract_entities_data(paths) + + assert not errors + assert [entity["key"] for entity in data] == ["dupe-key", "dupe-key"] + # Each one knows where it came from, which is what makes the eventual + # error message actionable. + assert [entity["src_path"] for entity in data] == paths + + def test_broken_files_are_collected_not_raised(self): + """ + extract_entities_data gathers per-file errors instead of raising. + + One unreadable entity shouldn't stop us reporting on the rest of them. + """ + data, errors = self.extractor.extract_entities_data( + ["broken.toml", "normal_container.toml"] + ) + + assert [type(err) for err in errors] == [payload.InvalidTOMLError] + assert errors[0].path == "broken.toml" + assert [entity["key"] for entity in data] == ["section-9-ac4b9f"] + + def test_ignore_unknown_tables(self): + """ + Allow for forwards compatibility. + + Unknown tables *inside* [entity] are preserved, so that a newer export + can add attributes without older code choking on them. Unknown tables at + the top level are not part of the entity, so they don't come along. + """ + data = self.extractor.extract_entity_data("unknown_table.toml") + assert data["future_thing"] == {"some_setting": "hello"} + assert "future_top_level" not in data + + def test_missing_versions(self): + """ + An entity with no [[version]] tables extracts to an empty version list. + + Whether that's actually loadable is the validation step's problem, not + ours -- our job is only to faithfully report what's in the file. + """ + data = self.extractor.extract_entity_data("missing_versions.toml") + assert data["key"] == "no-versions-c0ffee" + assert data["versions"] == [] + assert data["container"] == {"unit": {}} + + def test_normal_component(self): + data = self.extractor.extract_entity_data("normal_component.toml") + assert data["key"] == "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" + assert data["src_path"] == "normal_component.toml" + assert data["can_stand_alone"] is True + assert data["created"] == datetime( + 2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc + ) + assert data["draft"] == {"version_num": 3} + assert data["published"] == {"version_num": 2} + + # Components have no container table at all. + assert "container" not in data + + versions_by_num = {v["version_num"]: v for v in data["versions"]} + assert sorted(versions_by_num) == [2, 3] + + # Text media is inlined... + assert versions_by_num[2]["component"]["media"] == { + "block.xml": "

Version 2 text.

\n", + } + # ...while static assets are encoded as pointers back into the + # archive, so that we don't hold binary files in memory. + v3_media = versions_by_num[3]["component"]["media"] + assert v3_media["block.xml"] == "

Version 3 text.

\n" + assert v3_media["static/figure.png"] == "normal_component/component_versions/v3/static/figure.png" + + def test_normal_container(self): + data = self.extractor.extract_entity_data("normal_container.toml") + assert data == { + 'key': 'section-9-ac4b9f', + 'src_path': 'normal_container.toml', + 'can_stand_alone': True, + 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), + 'draft': { + 'version_num': 2 + }, + 'published': { + 'version_num': 1 + }, + 'container': { + 'section': {} + }, + 'versions': [ + { + 'title': 'Section 9', + 'version_num': 2, + 'container': { + 'children': [ + 'week-7-e73782', + 'subsection-001-e4bbe5' + ] + } + }, + { + 'title': 'Section 9', + 'version_num': 1, + 'container': { + 'children': [ + 'week-7-e73782' + ] + } + } + ] + } + + +class ExtractCollectionDataTest(TestCase): + """Tests for reading a single collection TOML file.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "collections") + cls.extractor = payload.PayloadExtractor(cls.fs) + + @classmethod + def tearDownClass(cls): + del cls.fs + del cls.extractor + super().tearDownClass() + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as ctx: + self.extractor.extract_collection_data("broken.toml") + assert ctx.exception.path == "broken.toml" + + def test_fields_not_in_table(self): + with self.assertRaises(payload.FieldsNotInTable) as ctx: + self.extractor.extract_collection_data("fields_not_in_table.toml") + assert ctx.exception.path == "fields_not_in_table.toml" + assert ctx.exception.fields == ["key", "title"] + + def test_missing_collection_table(self): + with self.assertRaises(payload.TableNotFoundError) as ctx: + self.extractor.extract_collection_data("missing_collection_table.toml") + assert ctx.exception.path == "missing_collection_table.toml" + assert ctx.exception.table == "collection" + assert "[collection]" in str(ctx.exception) + + def test_normal(self): + data = self.extractor.extract_collection_data("normal.toml") + assert data == { + "title": "Difficult Problems", + "key": "difficult-problems", + "description": "The tricky ones. 🐢", + "created": datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), + "entities": [ + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000001", + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000002", + ], + # Tracked so that validation errors can name the file they came from. + "src_path": "normal.toml", + } + + def test_dupes_are_not_caught_here(self): + """ + Duplicate Collection keys are a validation-level problem, not an + extraction-level one. + + Unlike entities -- which are assembled into a dict keyed by entity ref, + so a duplicate would silently overwrite -- collections are assembled into + a list. Nothing is lost at this layer, so we extract both and let + CompletePackageInputData.check_for_duplicate_keys reject them. + """ + dupe_1 = self.extractor.extract_collection_data("dupe_1.toml") + dupe_2 = self.extractor.extract_collection_data("dupe_2.toml") + assert dupe_1["key"] == dupe_2["key"] == "dupe-collection-key" + assert dupe_1["src_path"] != dupe_2["src_path"] + + +class ExtractUnvalidatedLearningPackageTest(TestCase): + """ + Tests for assembling a whole archive, rather than a single file. + + The important behavior here is that this function *collects* errors instead + of raising them, so that someone repairing an archive by hand sees every + problem at once. + """ + + def test_normal(self): + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + unvalidated = payload.PayloadExtractor(fs).extract() + + assert unvalidated.errors == [] + assert unvalidated.raw_data["learning_package"]["key"] == "lib:WGU:LIB_C001" + assert unvalidated.raw_data["meta"]["format_version"] == 1 + assert len(unvalidated.raw_data["entities"]) == 10 + assert len(unvalidated.raw_data["collections"]) == 1 + + def test_each_entity_records_its_own_source_file(self): + """ + src_path is the file, key is what's declared inside it. + + Two fixture files deliberately have names that don't match the key + inside them, because the export side hashes filenames to avoid + collisions. + """ + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + unvalidated = payload.PayloadExtractor(fs).extract() + + paths_by_key = { + entity["key"]: entity["src_path"] + for entity in unvalidated.raw_data["entities"] + } + assert ( + paths_by_key["section1-8ca126"] == "entities/section1-extra-8ca126.toml" + ) + assert paths_by_key[ + "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" + ].endswith("c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2-extra.toml") + + def test_missing_root_package_is_collected_not_raised(self): + fs = DirFileSystem(TEST_DATA_ROOT / "empty_archive") + unvalidated = payload.PayloadExtractor(fs).extract() + + assert len(unvalidated.errors) == 1 + error = unvalidated.errors[0] + assert isinstance(error, payload.MissingFileError) + assert error.path == "package.toml" + + # We still return a usable object, just an empty one. + assert unvalidated.raw_data["entities"] == [] + assert unvalidated.raw_data["collections"] == [] + + def test_duplicate_entities_both_survive_extraction(self): + """ + Extraction doesn't reject duplicates -- it preserves them for validation. + """ + fs = DirFileSystem(TEST_DATA_ROOT / "duplicate_entities") + unvalidated = payload.PayloadExtractor(fs).extract() + + # This fixture is only the two entity files, so the absent package.toml + # is expected. What matters is that the duplicate isn't an error here. + assert [type(err) for err in unvalidated.errors] == [payload.MissingFileError] + + entities = unvalidated.raw_data["entities"] + assert [entity["key"] for entity in entities] == ["dupe-key", "dupe-key"] + assert [entity["src_path"] for entity in entities] == [ + "entities/dupe_1.toml", + "entities/dupe_2.toml", + ] + + def test_static_assets_are_not_mistaken_for_entities(self): + """ + TOML files under component_versions/ are static assets, not entities. + """ + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + paths = payload.PayloadExtractor(fs).get_entity_file_paths() + + assert paths == sorted(paths) # deterministic ordering + assert all("/component_versions/" not in path for path in paths) + + def test_collection_errors_are_collected(self): + fs = DirFileSystem(TEST_DATA_ROOT / "broken_collection") + unvalidated = payload.PayloadExtractor(fs).extract() + + assert len(unvalidated.errors) == 1 + error = unvalidated.errors[0] + assert isinstance(error, payload.InvalidTOMLError) + assert error.path == "collections/broken.toml" + + # The rest of the archive still came through. + assert unvalidated.raw_data["learning_package"]["key"] == "lib:Axim:FunLib" + + +def dir_fs_with(tmp_path, layout: dict) -> DirFileSystem: + """ + Build a throwaway directory tree and return a filesystem over it. + + ``layout`` maps relative paths to file contents. + """ + for rel_path, contents in layout.items(): + target = Path(tmp_path) / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents) + return DirFileSystem(tmp_path) + + +def zip_fs_with(names) -> ZipFileSystem: + """Build an in-memory zip containing ``names``, and return a filesystem over it.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as zipf: + for name in names: + zipf.writestr(name, "[meta]\nformat_version = 1\n") + buffer.seek(0) + return ZipFileSystem(fo=buffer, mode="r") + + +class FindArchiveRootTest(TestCase): + """ + Tests for accepting archives that wrap their contents in a single folder. + + People routinely build an archive with ``zip -r MyLib.zip MyLib`` rather than + compressing the folder's *contents*, and the result is perfectly sensible to + them. We accept it rather than reporting a missing package.toml. + + Both a zip and a plain directory are covered, because fsspec's ``ls("")`` + behaves differently on each. + """ + + PACKAGE = "[meta]\nformat_version = 1\n" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = self._tmp.name + + def test_package_at_top_level_zip(self): + fs = zip_fs_with(["package.toml", "entities/unit1.toml"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_package_at_top_level_dir(self): + fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_single_wrapper_folder_zip(self): + fs = zip_fs_with(["MyLib/package.toml", "MyLib/entities/unit1.toml"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" + + def test_single_wrapper_folder_dir(self): + fs = dir_fs_with(self.tmp_path, {"MyLib/package.toml": self.PACKAGE}) + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" + + def test_macos_style_zip(self): + """ + macOS's "Compress" adds __MACOSX and often .DS_Store beside the folder. + + This is the single most common way a non-technical user produces a zip, + so a rule that just counted top-level entries would fail on most of them. + """ + fs = zip_fs_with([ + "MyLib/package.toml", + "MyLib/entities/unit1.toml", + "__MACOSX/._MyLib", + ".DS_Store", + ]) + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" + + def test_wrapper_folder_beside_a_stray_file(self): + fs = zip_fs_with(["MyLib/package.toml", "README.txt"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" + + def test_folder_without_a_package_toml_is_not_a_root(self): + """ + Requiring package.toml inside the candidate is what makes this safe. + + Without that check, any archive whose top level happened to hold a single + directory would be re-rooted into it. + """ + fs = zip_fs_with(["MyLib/entities/unit1.toml"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_two_candidate_folders_are_ambiguous(self): + fs = zip_fs_with(["LibA/package.toml", "LibB/package.toml"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_nested_two_levels_is_not_followed(self): + """We only look one level down; deeper nesting isn't worth guessing at.""" + fs = zip_fs_with(["Outer/MyLib/package.toml"]) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_empty_archive(self): + fs = dir_fs_with(self.tmp_path, {}) + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + + def test_entities_fixture_is_not_re_rooted(self): + """ + Regression guard for the fixture that nearly broke this. + + ``payload_test_data/entities/`` holds exactly one subdirectory, + ``normal_component/``. A rule based on "a single top-level folder" would + silently re-root into it and break every entity test in this module. + """ + fs = DirFileSystem(TEST_DATA_ROOT / "entities") + assert payload.PayloadExtractor.find_archive_root(fs) == "/" + assert payload.PayloadExtractor(fs).fs.path == "" + + +class AlwaysRerootedTest(TestCase): + """ + PayloadExtractor always wraps its source filesystem, wrapper or not. + + This is what lets ``fs.path`` be the single answer to "what did we treat as + the root?", which is why nothing stores that separately any more. Skipping + the wrap when no wrapper folder is found would pass every other test in this + module and quietly leave ``fs.path`` meaning two different things: "" for a + zip, but an absolute local path for a directory. + """ + + PACKAGE = "[meta]\nformat_version = 1\n" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = self._tmp.name + + def test_flat_zip_is_still_wrapped(self): + extractor = payload.PayloadExtractor(zip_fs_with(["package.toml"])) + + assert isinstance(extractor.fs, DirFileSystem) + assert extractor.fs.path == "" + + def test_flat_directory_is_still_wrapped(self): + source_fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) + extractor = payload.PayloadExtractor(source_fs) + + assert isinstance(extractor.fs, DirFileSystem) + # Not the absolute local path that source_fs is rooted at. + assert extractor.fs.path == "" + assert source_fs.path != "" + + def test_wrapped_archive_records_the_folder(self): + extractor = payload.PayloadExtractor(zip_fs_with(["MyLib/package.toml"])) + + assert isinstance(extractor.fs, DirFileSystem) + assert extractor.fs.path == "MyLib" + + def test_the_wrapper_is_a_pass_through_when_there_is_no_root(self): + """An identity wrap must not disturb any path operation.""" + source_fs = dir_fs_with(self.tmp_path, { + "package.toml": self.PACKAGE, + "entities/unit1.toml": "[entity]\nkey = \"unit1\"\n", + }) + extractor = payload.PayloadExtractor(source_fs) + + assert extractor.fs.exists("package.toml") + assert extractor.fs.glob("entities/*.toml") == ["entities/unit1.toml"] + assert extractor.fs.read_text("package.toml") == self.PACKAGE + + +class ExtractThroughWrapperTest(TestCase): + """ + Extracting a wrapper-style archive gives the same result as a flat one. + + Everything the extractor reports is relative to the detected root, so the + only observable difference should be ``root`` itself. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + flat_fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + cls.flat = payload.PayloadExtractor(flat_fs).extract() + + # The same fixture, but nested one level down inside "MyLib/". + wrapped_fs = DirFileSystem(FIXTURES_ROOT) + cls.wrapped_extractor = payload.PayloadExtractor(wrapped_fs) + + def test_flat_archive_has_no_root(self): + assert self.flat.fs.path == "" + + def test_wrapper_root_is_detected(self): + """ + ``fixtures/`` holds library_backup (with a package.toml) and broken/ + (without one), so library_backup is the only candidate. + """ + assert self.wrapped_extractor.fs.path == "library_backup" + + def test_raw_data_matches_the_flat_archive(self): + wrapped = self.wrapped_extractor.extract() + + assert wrapped.errors == [] + assert wrapped.raw_data == self.flat.raw_data + + def test_paths_are_relative_to_the_detected_root(self): + wrapped = self.wrapped_extractor.extract() + + paths_by_key = { + entity["key"]: entity["src_path"] + for entity in wrapped.raw_data["entities"] + } + assert paths_by_key["section1-8ca126"] == "entities/section1-extra-8ca126.toml" + + def test_static_asset_pointers_resolve_against_the_returned_fs(self): + """ + The file pointers are relative to the *re-rooted* filesystem. + + This is the subtle one: if extract() returned the original filesystem + instead of the re-rooted one, these pointers would not resolve, and + static assets would silently go missing for wrapper archives only. + """ + wrapped = self.wrapped_extractor.extract() + entity = next( + e for e in wrapped.raw_data["entities"] + if e["key"] == "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" + ) + version = next(v for v in entity["versions"] if v["version_num"] == 5) + pointer = version["component"]["media"]["static/me.png"] + + assert wrapped.fs.read_bytes(pointer) diff --git a/tests/openedx_content/applets/backup_restore/test_restore.py b/tests/openedx_content/applets/backup_restore/test_restore.py deleted file mode 100644 index fcaf99acd..000000000 --- a/tests/openedx_content/applets/backup_restore/test_restore.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Tests for the lp_load management command.""" -import os -from datetime import datetime, timezone -from io import StringIO -from unittest.mock import patch - -from django.contrib.auth import get_user_model -from django.core.management import call_command -from django.test import TestCase - -from openedx_content.applets.backup_restore.zipper import LearningPackageUnzipper, generate_staged_package_ref -from openedx_content.applets.collections import api as collections_api -from openedx_content.applets.components import api as components_api -from openedx_content.applets.containers import api as containers_api -from openedx_content.applets.publishing import api as publishing_api -from test_utils.zip_file_utils import folder_to_inmemory_zip - -User = get_user_model() - - -class RestoreTestCase(TestCase): - """Base test case for restore tests.""" - - def setUp(self): - super().setUp() - self.fixtures_folder = os.path.join(os.path.dirname(__file__), "fixtures/library_backup") - self.zip_file = folder_to_inmemory_zip(self.fixtures_folder) - self.package_ref = "lib:WGU:LIB_C001" - self.user = User.objects.create_user(username='lp_user', password='12345') - - -class RestoreLearningPackageCommandTest(RestoreTestCase): - """Tests for the lp_load management command.""" - - @patch("openedx_content.applets.backup_restore.api.load_learning_package") - def test_restore_command(self, mock_load_learning_package): - # Mock load_learning_package to return our in-memory zip file - restore_result = LearningPackageUnzipper(self.zip_file, user=self.user).load() - mock_load_learning_package.return_value = restore_result - - out = StringIO() - # You can pass any dummy path, since load_learning_package is mocked - call_command("lp_load", "dummy.zip", "lp_user", stdout=out) - - lp = self.verify_lp(restore_result["lp_restored_data"]["package_ref"]) - self.verify_containers(lp) - self.verify_components(lp) - self.verify_collections(lp) - - def verify_lp(self, package_ref): - """Verify the learning package was restored correctly.""" - lp = publishing_api.LearningPackage.objects.filter(package_ref=package_ref).first() - assert lp is not None, "Learning package was not restored." - assert lp.title == "Library test" - assert lp.description == "" - return lp - - def verify_containers(self, lp): - """Verify the containers and their versions were restored correctly.""" - container_qs = containers_api.get_containers(learning_package_id=lp.id) - expected_container_keys = ["unit1-b7eafb", "subsection1-48afa3", "section1-8ca126"] - - for container in container_qs: - assert container.entity_ref in expected_container_keys - draft_version = publishing_api.get_draft_version(container.publishable_entity.id) - published_version = publishing_api.get_published_version(container.publishable_entity.id) - assert container.created_by is not None - assert container.created_by.username == "lp_user" - # The unit has been published. The other two containers haven't been. - # It's important that we test with at least one published container in order to - # fully cover _create_container in zipper.py. - if container.entity_ref == "unit1-b7eafb": - assert containers_api.get_container_type_code_of(container) == "unit" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is not None - assert published_version.version_num == 2 - assert published_version.created_by is not None - assert published_version.created_by.username == "lp_user" - elif container.entity_ref == "subsection1-48afa3": - assert containers_api.get_container_type_code_of(container) == "subsection" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - elif container.entity_ref == "section1-8ca126": - assert containers_api.get_container_type_code_of(container) == "section" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - else: - assert False, f"Unexpected container key: {container.entity_ref}" - - def verify_components(self, lp): - # pylint: disable=too-many-statements - """Verify the components and their versions were restored correctly.""" - component_qs = components_api.get_components(lp.id) - expected_component_keys = [ - "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27", - "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37", - "xblock.v1:openassessment:1ee38208-a585-4455-a27e-4930aa541f53", - "xblock.v1:problem:256739e8-c2df-4ced-bd10-8156f6cfa90b", - "xblock.v1:survey:6681da3f-b056-4c6e-a8f9-040967907471", - "xblock.v1:video:22601ebd-9da8-430b-9778-cfe059a98568", - "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" - ] - for component in component_qs: - assert component.entity_ref in expected_component_keys - draft_version = publishing_api.get_draft_version(component.publishable_entity.id) - published_version = publishing_api.get_published_version(component.publishable_entity.id) - assert component.created_by is not None - assert component.created_by.username == "lp_user" - if component.entity_ref == "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27": - assert component.component_type.name == "drag-and-drop-v2" - assert component.component_type.namespace == "xblock.v1" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - # Get the content associated with this component - all_media = draft_version.componentversion.media.all() - media = all_media.first() if all_media.exists() else None - assert media is not None - assert " dict: + """The smallest input we consider loadable.""" + return { + "meta": {"format_version": 1}, + "learning_package": {"key": "lib:Axim:FunLib"}, + "entities": [], + "collections": [], + **overrides, + } + + +class ContainerDiscriminationTest(TestCase): + """ + The container union has to resolve to the right model. + + All three container models allow extra fields, so if the discriminating field + were optional every one of them would validate every dict, and the union + would always collapse to whichever model happens to be listed first. That + failure is silent -- everything would load as a Unit -- so it's worth pinning + down explicitly. + """ + + def _container_for(self, raw): + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED, "container": raw}) + return entity.container + + def test_section(self): + assert isinstance(self._container_for({"section": {}}), SectionInputData) + + def test_subsection(self): + assert isinstance(self._container_for({"subsection": {}}), SubsectionInputData) + + def test_unit(self): + assert isinstance(self._container_for({"unit": {}}), UnitInputData) + + def test_unknown_container_type_falls_through_to_dict(self): + """ + A container type we don't recognize is still captured. + + We keep the raw dict rather than erroring here so that validation can + report a useful message. Loading rejects it. + """ + container = self._container_for({"chapter": {}}) + assert isinstance(container, dict) + assert container == {"chapter": {}} + + def test_no_container_means_component(self): + assert self._container_for(None) is None + + def test_absent_container_defaults_to_none(self): + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED}) + assert entity.container is None + + +class StringConstraintsTest(TestCase): + """ + The ref/code constraints have to actually be applied. + + These were declared with a trailing comma at one point, which made them + tuples rather than StringConstraints, and pydantic silently ignored them. + """ + + def test_collection_key_is_stripped(self): + collection = CollectionInput.model_validate( + {"title": "Difficult Problems", "key": " difficult-problems "} + ) + assert collection.key == "difficult-problems" + + def test_collection_key_rejects_spaces(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "has spaces"}) + + def test_collection_key_rejects_slashes(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "has/slash"}) + + def test_collection_key_rejects_trailing_newline(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "trailing\nnewline"}) + + def test_learning_package_key_is_stripped(self): + lp = LearningPackageInputData.model_validate({"key": " lib:Axim:FunLib "}) + assert lp.key == "lib:Axim:FunLib" + + +class MetaInputDataTest(TestCase): + """Tests for the archive provenance metadata.""" + + def test_only_format_version_is_required(self): + """ + Everything else in [meta] is provenance information we can't trust + anyway, so an archive that omits it is still loadable. + """ + meta = MetaInputData.model_validate({"format_version": 1}) + + assert meta.format_version == 1 + assert meta.created_by is None + assert meta.created_by_email is None + assert meta.created_at is None + assert meta.origin_server is None + + def test_format_version_is_required(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate({}) + + def test_format_version_must_be_1(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate({"format_version": 2}) + + def test_email_is_validated(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate( + {"format_version": 1, "created_by_email": "not-an-email"} + ) + + +class LearningPackageInputDataTest(TestCase): + """Tests for the top-level Learning Package fields.""" + + def test_key_is_required(self): + with self.assertRaises(ValidationError) as ctx: + LearningPackageInputData.model_validate({"title": "Fun Library"}) + assert [err["loc"] for err in ctx.exception.errors()] == [("key",)] + + def test_title_has_a_default(self): + lp = LearningPackageInputData.model_validate({"key": "lib:Axim:FunLib"}) + assert lp.title == "Untitled Library" + + def test_blank_title_is_rejected(self): + with self.assertRaises(ValidationError): + LearningPackageInputData.model_validate({"key": "lib:A:B", "title": ""}) + + def test_dates_are_optional(self): + lp = LearningPackageInputData.model_validate({"key": "lib:Axim:FunLib"}) + assert lp.created is None + assert lp.updated is None + + def test_naive_datetimes_are_rejected(self): + """We store everything in UTC, so an ambiguous timestamp is an error.""" + with self.assertRaises(ValidationError): + LearningPackageInputData.model_validate( + {"key": "lib:A:B", "created": datetime(2026, 4, 8, 15, 22, 12)} + ) + + +class VersionInputTest(TestCase): + """Tests for entity versions and their draft/published pointers.""" + + def _entity_with_version(self, **version_overrides): + version = {"version_num": 1, "title": "Some Title", **version_overrides} + return EntityInputData.model_validate( + {"key": "some-entity", "created": CREATED, "versions": [version]} + ) + + def test_blank_title_is_allowed(self): + """ + Blank titles are legal and common -- content imported from courses (e.g. + via the modulestore migrator) frequently has untitled units, and such + content can be backed up. Restoring that same archive must work. + """ + entity = self._entity_with_version(title="") + assert entity.versions[0].title == "" + + def test_version_num_must_be_positive(self): + with self.assertRaises(ValidationError): + self._entity_with_version(version_num=0) + + def test_draft_and_published_default_to_none(self): + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED}) + assert entity.draft.version_num is None + assert entity.published.version_num is None + + def test_empty_published_table_means_unpublished(self): + """ + An entity that has never been published exports an empty + [entity.published] table rather than omitting it. + """ + entity = EntityInputData.model_validate( + {"key": "some-entity", "created": CREATED, "published": {}, "draft": {"version_num": 2}} + ) + assert entity.published.version_num is None + assert entity.draft.version_num == 2 + + +class CompletePackageInputDataTest(TestCase): + """Tests for validating a whole package document at once.""" + + def test_minimal_package(self): + data = CompletePackageInputData.model_validate(minimal_package()) + + assert data.learning_package.key == "lib:Axim:FunLib" + assert data.entities == [] + assert data.collections == [] + + def test_duplicate_collection_keys_are_rejected(self): + raw = minimal_package( + collections=[ + {"title": "One", "key": "same-key", "src_path": "collections/a.toml"}, + {"title": "Two", "key": "same-key", "src_path": "collections/b.toml"}, + ] + ) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + message = str(ctx.exception) + assert "same-key" in message + # The message should name both files, so it's actionable. + assert "collections/a.toml" in message + assert "collections/b.toml" in message + + def test_distinct_collection_keys_are_fine(self): + raw = minimal_package( + collections=[ + {"title": "One", "key": "key-one"}, + {"title": "Two", "key": "key-two"}, + ] + ) + data = CompletePackageInputData.model_validate(raw) + assert [c.key for c in data.collections] == ["key-one", "key-two"] + + def test_collection_entities_default_to_empty(self): + raw = minimal_package(collections=[{"title": "One", "key": "key-one"}]) + data = CompletePackageInputData.model_validate(raw) + assert data.collections[0].entities == [] + + def test_missing_meta_reports_against_meta(self): + raw = minimal_package() + del raw["meta"] + + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + assert ("meta",) in [err["loc"] for err in ctx.exception.errors()] + + def test_errors_are_reported_together(self): + """ + Pydantic collects every problem, which is what lets us show someone + repairing an archive all of their mistakes at once. + """ + raw = {"meta": {}, "learning_package": {}, "entities": {}, "collections": []} + + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + locations = {err["loc"] for err in ctx.exception.errors()} + assert ("meta", "format_version") in locations + assert ("learning_package", "key") in locations + + +class ForwardsCompatibilityTest(TestCase): + """ + Unrecognized fields are kept rather than dropped. + + Older installs need to load newer archives. We keep the unknown values rather + than ignoring them so that we can eventually warn about fields that look like + typos of ones we do know. + """ + + def test_unknown_fields_are_retained(self): + data = CompletePackageInputData.model_validate( + minimal_package( + learning_package={"key": "lib:A:B", "some_future_field": "hello"} + ) + ) + assert data.learning_package.some_future_field == "hello" + + def test_models_are_frozen(self): + data = CompletePackageInputData.model_validate(minimal_package()) + with self.assertRaises(ValidationError): + data.learning_package.title = "Changed" + + +class BlankMetadataTest(TestCase): + """ + Blank [meta] values must not block a restore. + + The backup side writes ``created_by_email`` unconditionally, and Django's + ``User.email`` defaults to an empty string, so archives with blank + provenance fields are routinely produced by our own export. + """ + + def test_blank_email(self): + meta = MetaInputData.model_validate( + {"format_version": 1, "created_by_email": ""} + ) + assert meta.created_by_email is None + + def test_blank_created_by(self): + meta = MetaInputData.model_validate({"format_version": 1, "created_by": " "}) + assert meta.created_by is None + + def test_blank_origin_server(self): + meta = MetaInputData.model_validate({"format_version": 1, "origin_server": ""}) + assert meta.origin_server is None + + def test_real_values_still_come_through(self): + meta = MetaInputData.model_validate({ + "format_version": 1, + "created_by": "eddy", + "created_by_email": "eddy@axim.org", + "origin_server": "studio.local.openedx.io", + }) + assert meta.created_by == "eddy" + assert meta.created_by_email == "eddy@axim.org" + assert meta.origin_server == "studio.local.openedx.io" + + def test_a_genuinely_bad_email_is_still_rejected(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate( + {"format_version": 1, "created_by_email": "not-an-email"} + ) + + +class DuplicateEntityKeyTest(TestCase): + """ + Entities are validated for duplicate keys the same way collections are. + + Both are declared as lists rather than dicts keyed by their key field, + precisely so that a redefinition survives extraction and can be reported + here instead of silently overwriting its predecessor. + """ + + def entity(self, key, src_path): + return {"key": key, "src_path": src_path, "created": CREATED} + + def test_duplicate_entity_keys_are_rejected(self): + raw = minimal_package(entities=[ + self.entity("unit1-b7eafb", "entities/first.toml"), + self.entity("unit1-b7eafb", "entities/second.toml"), + ]) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + message = str(ctx.exception) + assert "unit1-b7eafb" in message + assert "entities/first.toml" in message + assert "entities/second.toml" in message + + def test_distinct_entity_keys_are_fine(self): + raw = minimal_package(entities=[ + self.entity("unit1-b7eafb", "entities/first.toml"), + self.entity("unit2-c9dfa1", "entities/second.toml"), + ]) + data = CompletePackageInputData.model_validate(raw) + assert [e.key for e in data.entities] == ["unit1-b7eafb", "unit2-c9dfa1"] + + def test_key_is_required(self): + raw = minimal_package(entities=[{"created": CREATED}]) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + assert ("entities", 0, "key") in [err["loc"] for err in ctx.exception.errors()] + + def test_src_path_is_optional(self): + """It's only ever used to build error messages.""" + raw = minimal_package(entities=[{"key": "unit1-b7eafb", "created": CREATED}]) + data = CompletePackageInputData.model_validate(raw) + assert data.entities[0].src_path is None diff --git a/tests/openedx_content/applets/backup_restore/test_validation.py b/tests/openedx_content/applets/backup_restore/test_validation.py new file mode 100644 index 000000000..07319798b --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_validation.py @@ -0,0 +1,479 @@ +""" +Tests for turning extracted archive data into a validated model. + +The contract this module has to hold up: + +* ``validate`` never raises. Everything it finds goes onto ``.errors``, so that + someone repairing an archive by hand sees every problem in one pass. +* Errors name the archive file they came from. Pydantic reports against the + combined document we assemble internally, which nobody editing an archive has + ever seen, so we translate those locations back into file paths. +* Every cross-reference that ``loading.py`` relies on is checked here, so that a + broken archive produces a readable error instead of a traceback from the + middle of a database write. + +These tests are strictly for the validation module, and therefore don't need +Django to run. +""" +from datetime import datetime, timezone +from pathlib import Path +from unittest import TestCase + +from fsspec.implementations.dirfs import DirFileSystem + +from openedx_content.applets.backup_restore import validation +from openedx_content.applets.backup_restore.errors import ( + DuplicateVersionError, + InvalidTOMLError, + MalformedRefError, + MissingFileError, + MissingVersionError, + RestoreFailedError, + SchemaError, + UnknownContainerTypeError, + UnresolvedChildError, +) +from openedx_content.applets.backup_restore.payload import UnvalidatedLearningPackageInput + +FIXTURES_ROOT = Path(__file__).parent / "fixtures" +CREATED = datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc) + + +def component(key: str, **overrides) -> dict: + """ + Raw data for a Component entity (i.e. one with no container). + + Each entity carries its own key and source file, the same way collections + do, which is what lets validation report duplicates and attribute errors to + a file. + """ + return { + "key": key, + "src_path": f"entities/{key}.toml", + "can_stand_alone": True, + "created": CREATED, + "versions": [], + **overrides, + } + + +def container(key: str, kind: str, **overrides) -> dict: + """Raw data for a Container entity of the given kind.""" + return component(key, container={kind: {}}, **overrides) + + +def version(version_num: int, children=None, **overrides) -> dict: + raw = {"version_num": version_num, "title": f"v{version_num}", **overrides} + if children is not None: + raw["container"] = {"children": children} + return raw + + +def unvalidated(entities=None, collections=None, errors=None, root=None, **overrides): + """Build an UnvalidatedLearningPackageInput without going through files.""" + raw_data = { + "meta": {"format_version": 1}, + "learning_package": {"key": "lib:Axim:FunLib"}, + "entities": entities if entities is not None else [], + "collections": collections if collections is not None else [], + **overrides, + } + return UnvalidatedLearningPackageInput( + raw_data=raw_data, + errors=errors or [], + # Built the way PayloadExtractor builds it: always a DirFileSystem, whose + # .path is the wrapper folder inside the archive (or "" if there is none). + fs=DirFileSystem(path=root or "/", fs=DirFileSystem(FIXTURES_ROOT)), + ) + + +class ValidateSuccessTest(TestCase): + """Archives that validate cleanly.""" + + def test_minimal_package(self): + result = validation.validate(unvalidated()) + + assert result.errors == [] + assert result.data is not None + assert result.data.learning_package.key == "lib:Axim:FunLib" + + def test_fs_is_passed_through(self): + """The loader needs the filesystem later, to read static assets.""" + source = unvalidated() + result = validation.validate(source) + assert result.fs is source.fs + + def test_well_formed_container_tree(self): + result = validation.validate(unvalidated(entities=[ + container("section-1", "section", versions=[version(1, ["subsection-1"])]), + container("subsection-1", "subsection", versions=[version(1, ["unit-1"])]), + container("unit-1", "unit", versions=[version(1, ["xblock.v1:html:abc"])]), + component("xblock.v1:html:abc", versions=[version(1)]), + ])) + + assert result.errors == [] + + +class ExtractionErrorsCarryThroughTest(TestCase): + """ + Extraction errors are already BackupRestoreErrors, so they come through as + themselves -- not flattened into some other error record. + """ + + def test_error_type_and_path_are_preserved(self): + extraction_error = InvalidTOMLError( + "Entity", details="bad token", path="entities/broken.toml" + ) + result = validation.validate(unvalidated(errors=[extraction_error])) + + assert result.errors[0] is extraction_error + assert isinstance(result.errors[0], InvalidTOMLError) + assert result.errors[0].path == "entities/broken.toml" + + def test_extraction_errors_are_reported_alongside_schema_errors(self): + result = validation.validate( + unvalidated( + errors=[MissingFileError("Root Package", path="package.toml")], + learning_package={}, # also missing its key + ) + ) + + error_types = {type(err) for err in result.errors} + assert MissingFileError in error_types + assert SchemaError in error_types + + +class SchemaErrorSourceMappingTest(TestCase): + """ + Pydantic's `loc` gets translated back into "which file is wrong". + """ + + def test_learning_package_errors_point_at_package_toml(self): + result = validation.validate(unvalidated(learning_package={})) + + assert result.data is None + error = result.errors[0] + assert isinstance(error, SchemaError) + assert error.path == "package.toml" + assert error.location == ("learning_package", "key") + + def test_meta_errors_point_at_package_toml(self): + result = validation.validate(unvalidated(meta={})) + + error = result.errors[0] + assert error.path == "package.toml" + assert error.location == ("meta", "format_version") + + def test_entity_errors_point_at_the_entity_file(self): + result = validation.validate(unvalidated( + entities=[container("unit1-b7eafb", "unit", versions=[{"version_num": 1}])], + )) + + error = result.errors[0] + assert error.path == "entities/unit1-b7eafb.toml" + # The list index is dropped, since the file only describes one entity. + assert error.location == ("versions", 0, "title") + + def test_entity_errors_fall_back_when_path_is_unknown(self): + """An entity with no recorded source file still gets its error reported.""" + result = validation.validate(unvalidated( + entities=[ + container("unit-1", "unit", versions=[{"version_num": 1}], src_path=None) + ], + )) + + assert result.errors[0].path is None + assert result.errors[0].location == ("versions", 0, "title") + + def test_collection_errors_point_at_the_collection_file(self): + result = validation.validate(unvalidated( + collections=[{"key": "no-title", "src_path": "collections/broken.toml"}], + )) + + error = result.errors[0] + assert error.path == "collections/broken.toml" + assert error.location == ("title",) + + def test_error_message_includes_location(self): + result = validation.validate(unvalidated( + entities=[container("unit1", "unit", versions=[{"version_num": 1}])], + )) + + assert "entities/unit1.toml: versions.0.title" in str(result.errors[0]) + + +class ConsistencyCheckTest(TestCase): + """ + Cross-references pydantic can't express. Each of these would otherwise be an + uncaught exception in the middle of loading. + """ + + def test_unresolved_child(self): + result = validation.validate(unvalidated( + entities=[container("unit1", "unit", versions=[version(1, ["nope"])])], + )) + + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, UnresolvedChildError) + assert error.path == "entities/unit1.toml" + assert "nope" in error.message + + def test_draft_pointing_at_a_missing_version(self): + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", draft={"version_num": 7}, versions=[version(1)]), + ])) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MissingVersionError) + assert "[entity.draft]" in result.errors[0].message + + def test_published_pointing_at_a_missing_version(self): + result = validation.validate(unvalidated(entities=[ + container( + "unit-1", "unit", published={"version_num": 7}, versions=[version(1)] + ), + ])) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MissingVersionError) + assert "[entity.published]" in result.errors[0].message + + def test_draft_and_published_may_be_absent(self): + """An entity that was created and then reset to published has neither.""" + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", versions=[version(1)]), + ])) + assert result.errors == [] + + def test_duplicate_version_num(self): + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", versions=[version(2), version(2)]), + ])) + + duplicate_errors = [ + err for err in result.errors if isinstance(err, DuplicateVersionError) + ] + assert len(duplicate_errors) == 1 + assert "version 2" in duplicate_errors[0].message + + def test_malformed_component_ref(self): + """ + Component refs are "{namespace}:{type}:{code}" -- the loader splits on + the colons to work out what kind of block to build. + """ + result = validation.validate(unvalidated(entities=[ + component("not-a-component-ref", versions=[version(1)]), + ])) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MalformedRefError) + + def test_container_refs_are_not_required_to_have_colons(self): + """Only Components derive meaning from the shape of their ref.""" + result = validation.validate(unvalidated(entities=[ + container("unit1-b7eafb", "unit", versions=[version(1)]), + ])) + assert result.errors == [] + + def test_unknown_container_type(self): + result = validation.validate(unvalidated( + entities=[component("thing1", container={"chapter": {}})], + )) + + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, UnknownContainerTypeError) + assert error.path == "entities/thing1.toml" + assert "chapter" in error.message + + def test_consistency_checks_are_skipped_when_the_schema_is_broken(self): + """ + If we couldn't build a model, there's nothing to cross-reference. We + report the schema errors rather than inventing consistency errors on top + of data we know is malformed. + """ + result = validation.validate(unvalidated( + meta={}, + entities=[container("unit-1", "unit", versions=[version(1, ["nope"])])], + )) + + assert result.data is None + assert all(isinstance(err, SchemaError) for err in result.errors) + + def test_all_problems_are_reported_together(self): + result = validation.validate(unvalidated(entities=[ + container( + "unit-1", "unit", draft={"version_num": 9}, versions=[version(1, ["nope"])] + ), + component("bad-component-ref", versions=[version(1)]), + ])) + + error_types = {type(err) for err in result.errors} + assert error_types == { + UnresolvedChildError, + MissingVersionError, + MalformedRefError, + } + + +class RestoreFailedErrorTest(TestCase): + """Tests for how a batch of errors is reported.""" + + def test_as_text_matches_the_legacy_log_format(self): + error = RestoreFailedError([ + MissingFileError("Root Package", path="package.toml"), + InvalidTOMLError("Entity", details="bad token", path="entities/x.toml"), + ]) + + assert error.as_text() == ( + "Errors encountered during restore:\n" + "package.toml: Root Package file not found at expected path\n" + "entities/x.toml: Cannot decode TOML for Entity: bad token\n" + ) + + def test_errors_are_kept_for_inspection(self): + original = MissingFileError("Root Package", path="package.toml") + error = RestoreFailedError([original]) + + assert error.errors == [original] + + +class SourceMappingFallbackTest(TestCase): + """ + Errors we can't attribute to a file still get reported. + + These paths shouldn't come up in practice, but silently dropping an error + because we couldn't work out where it came from would be much worse than + reporting it without a filename. + """ + + def test_unrecognized_location_has_no_path(self): + result = validation.validate(unvalidated(entities="not-a-list")) + + assert result.data is None + error = result.errors[0] + assert error.path is None + assert error.location == ("entities",) + assert str(error).startswith("entities:") + + def test_collection_without_a_src_path(self): + result = validation.validate(unvalidated( + collections=[{"key": "no-title"}], # no src_path to attribute it to + )) + + assert result.errors[0].path is None + + def test_schema_error_without_a_location(self): + error = SchemaError("something went wrong", path="package.toml") + assert str(error) == "package.toml: something went wrong" + + +class ArchiveRootPassthroughTest(TestCase): + """ + The detected archive root travels with the validated input. + + It has no effect on validation -- every path is already relative to it -- but + the error report says which folder we picked, since that isn't obvious from + the paths alone. + """ + + def test_root_is_carried_through(self): + """ + The root travels on the filesystem itself, not as a separate field. + + DirFileSystem already records what it was rooted at, so there is nothing + for validation to copy across. + """ + assert validation.validate(unvalidated(root="MyLib")).fs.path == "MyLib" + + def test_no_root_by_default(self): + assert validation.validate(unvalidated()).fs.path == "" + + def test_as_text_names_the_root_when_there_is_one(self): + error = RestoreFailedError( + [MissingFileError("Root Package", path="package.toml")], + archive_root="MyLib", + ) + + assert error.as_text() == ( + "Errors encountered during restore:\n" + "Archive root: MyLib/\n" + "package.toml: Root Package file not found at expected path\n" + ) + + def test_as_text_omits_the_root_when_there_isn_t_one(self): + error = RestoreFailedError([MissingFileError("Root Package", path="package.toml")]) + + assert "Archive root" not in error.as_text() + + +class DuplicateEntityTest(TestCase): + """ + Two files defining the same entity key. + + This is checked here rather than during extraction for the same reason + duplicate Collections are: entities are kept in a list, so both definitions + survive extraction intact and validation can name both files. + """ + + def test_duplicate_entity_keys_are_rejected(self): + result = validation.validate(unvalidated(entities=[ + component("xblock.v1:html:abc", src_path="entities/first.toml"), + component("xblock.v1:html:abc", src_path="entities/second.toml"), + ])) + + assert result.data is None + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, SchemaError) + # The message has to name both files to be actionable. + assert "entities/first.toml" in error.message + assert "entities/second.toml" in error.message + assert "xblock.v1:html:abc" in error.message + + def test_distinct_entity_keys_are_fine(self): + result = validation.validate(unvalidated(entities=[ + component("xblock.v1:html:abc"), + component("xblock.v1:html:def"), + ])) + + assert result.errors == [] + + def test_missing_entity_key(self): + """ + An entity file with no key at all. + + Extraction used to reject this, because the key became a dict key and it + had nowhere to put an entity without one. Now it's a required field like + any other. + """ + result = validation.validate(unvalidated( + entities=[{"created": CREATED, "src_path": "entities/no_key.toml"}], + )) + + assert result.data is None + error = result.errors[0] + assert isinstance(error, SchemaError) + assert error.path == "entities/no_key.toml" + assert error.location == ("key",) + + +class ErrorTextTest(TestCase): + """How individual errors render into the restore log.""" + + def test_an_error_with_no_path_omits_it(self): + """ + Not everything is attributable to a single file. + + A duplicate key is reported against the whole section, so there is no + one path to name -- and "None: ..." in a log file helps nobody. + """ + error = UnknownContainerTypeError("Entity declares an unsupported container") + + assert str(error) == "Entity declares an unsupported container" + + def test_an_error_with_a_path_names_it(self): + error = UnknownContainerTypeError("nope", path="entities/thing.toml") + + assert str(error) == "entities/thing.toml: nope" diff --git a/tox.ini b/tox.ini index ad8736086..2f3bb87bb 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,10 @@ match-dir = (?!migrations) [pytest] DJANGO_SETTINGS_MODULE = test_settings +; So that a bare `pytest` run (outside tox, which installs the package) can +; still find the source tree. +pythonpath = src +testpaths = tests addopts = --cov src --cov tests --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements site-packages filterwarnings =