From 482b59d20fa1802ee3cbd31d8e676e9df9471fdd Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 7 Apr 2026 21:43:18 -0400 Subject: [PATCH 01/14] feat: switch LearningPackage restore to pydantic This is a re-implementation of the restore part of backup_restore, with the goal of making it more robust and maintainable in the long term. --- .../decisions/0025-backup-restore.rst | 46 ++ pytest.ini | 4 + requirements/base.in | 4 + requirements/base.txt | 32 +- requirements/ci.txt | 8 +- requirements/dev.txt | 49 +- requirements/doc.txt | 41 +- requirements/pip-tools.txt | 4 +- requirements/quality.txt | 41 +- requirements/test.txt | 37 +- .../applets/backup_restore/api.py | 111 +++- .../applets/backup_restore/archive.py | 32 ++ .../applets/backup_restore/loading.py | 326 +++++++++++ .../applets/backup_restore/payload.py | 511 ++++++++++++++++++ .../applets/backup_restore/readme.rst | 30 + .../applets/backup_restore/schema.py | 302 +++++++++++ .../applets/backup_restore/validation.py | 39 ++ .../management/commands/encode.py | 98 ++++ .../management/commands/lp_load2.py | 67 +++ .../payload_test_data/collections/broken.toml | 0 .../payload_test_data/collections/dupe_1.toml | 0 .../payload_test_data/collections/dupe_2.toml | 0 .../collections/fields_not_in_table.toml | 0 .../collections/missing_collection_table.toml | 0 .../payload_test_data/entities/broken.toml | 2 + .../payload_test_data/entities/dupe_1.toml | 4 + .../payload_test_data/entities/dupe_2.toml | 4 + .../entities/missing_entity_key.toml | 10 + .../entities/missing_entity_table.toml | 11 + .../entities/missing_versions.toml | 0 .../entities/normal_component.toml | 0 .../entities/normal_container.toml | 29 + .../root_packages/broken.toml | 3 + .../root_packages/fields_not_in_table.toml | 15 + .../root_packages/minimal.toml | 9 + .../missing_learning_package.toml | 3 + .../root_packages/missing_meta.toml | 4 + .../root_packages/normal_ulmo_v1.toml | 15 + .../root_packages/unknown_table.toml | 10 + .../unsupported_format_version_1_1.toml | 7 + .../unsupported_format_version_2.toml | 6 + .../unsupported_format_version_b.toml | 6 + .../applets/backup_restore/test_payload.py | 229 ++++++++ .../applets/backup_restore/test_restore.py | 12 +- 44 files changed, 2089 insertions(+), 72 deletions(-) create mode 100644 docs/openedx_content/decisions/0025-backup-restore.rst create mode 100644 pytest.ini create mode 100644 src/openedx_content/applets/backup_restore/archive.py create mode 100644 src/openedx_content/applets/backup_restore/loading.py create mode 100644 src/openedx_content/applets/backup_restore/payload.py create mode 100644 src/openedx_content/applets/backup_restore/readme.rst create mode 100644 src/openedx_content/applets/backup_restore/schema.py create mode 100644 src/openedx_content/applets/backup_restore/validation.py create mode 100644 src/openedx_content/management/commands/encode.py create mode 100644 src/openedx_content/management/commands/lp_load2.py create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml create mode 100644 tests/openedx_content/applets/backup_restore/test_payload.py diff --git a/docs/openedx_content/decisions/0025-backup-restore.rst b/docs/openedx_content/decisions/0025-backup-restore.rst new file mode 100644 index 000000000..be3b636f9 --- /dev/null +++ b/docs/openedx_content/decisions/0025-backup-restore.rst @@ -0,0 +1,46 @@ +25. Learning Package Serialization and Validation Approach +========================================================== + +Context +------- + +Content Libraries map 1:1 to LearningPackages and these need to be imported and exported as file archives. Initial support for this was released in Ulmo, but we wanted to revisit it to make it more robust during the Verawood timeline. This is part of that effort. + +* Flexibility of Structure +* Standardization of validation (JSON Schema) +* Justify ZIP +* Justify TOML +* Max 100,000 items. +* Use of fsspec as abstraction + +Phases + +Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage + + +Decision +-------- + +Some key points: + +1. We intentionally separate input and output formats because the output format + will change over time, but the various input formats must continue to be + supported. We don't inherit from one from the other because we don't *want* + those changes to be automatically propogated--that breaks compatibility. +2. We assemble into giant JSON in order to simplify validation and allow for + more flexibility in structural representation. There's the archive layer and + then the logical layer and then serialization into the database. + + +Archive -> Model (validation) + Resources -> Database + + + +Consequences +------------ + + + +Rejected alternatives +--------------------- + diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..3621a80a5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = ["tests"] +pythonpath = src +DJANGO_SETTINGS_MODULE = test_settings diff --git a/requirements/base.in b/requirements/base.in index 626a551be..a384a92af 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -17,3 +17,7 @@ rules<4.0 # Django extension for rules-based authorization check tomlkit # Parses and writes TOML configuration files edx-organizations # Implemented the "Organization" model that CatalogCourse/CourseRun are keyed to + +fsspec # Used by openedx_content's backup_restore to abstract zip access + +pydantic[email] # Used by openedx_content's backup_restore for input validation diff --git a/requirements/base.txt b/requirements/base.txt index 43659d809..1cd2dee59 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,6 +6,8 @@ # amqp==5.3.1 # via kombu +annotated-types==0.8.0 + # via pydantic asgiref==3.12.1 # via django attrs==26.1.0 @@ -24,7 +26,7 @@ cffi==2.1.1 # pynacl charset-normalizer==3.5.1 # via requests -click==8.4.2 +click==8.5.0 # via # celery # click-didyoumean @@ -37,7 +39,7 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -cryptography==50.0.0 +cryptography==50.0.1 # via pyjwt django==5.2.17 # via @@ -70,7 +72,9 @@ djangorestframework==3.18.0 # edx-drf-extensions # edx-organizations dnspython==2.8.0 - # via pymongo + # via + # email-validator + # pymongo drf-jwt==1.19.2 # via edx-drf-extensions edx-ccx-keys==2.0.2 @@ -91,10 +95,16 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/base.in +email-validator==2.3.0 + # via pydantic fastavro==1.12.2 # via openedx-events +fsspec==2026.7.0 + # via -r requirements/base.in idna==3.19 - # via requests + # via + # email-validator + # requests kombu==5.6.2 # via celery openedx-events==11.2.0 @@ -109,6 +119,10 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi +pydantic[email]==2.13.5 + # via -r requirements/base.in +pydantic-core==2.46.5 + # via pydantic pyjwt[crypto]==2.13.0 # via # drf-jwt @@ -138,7 +152,13 @@ stevedore==5.9.1 tomlkit==0.15.1 # via -r requirements/base.in typing-extensions==4.16.0 - # via edx-opaque-keys + # via + # edx-opaque-keys + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via pydantic tzdata==2026.3 # via kombu tzlocal==5.4.4 @@ -150,7 +170,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via prompt-toolkit # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/ci.txt b/requirements/ci.txt index 8ba2a927b..044e40258 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -19,7 +19,7 @@ packaging==26.3 # via # pyproject-api # tox -platformdirs==4.11.3 +platformdirs==4.11.5 # via # tox # virtualenv @@ -27,15 +27,15 @@ pluggy==1.6.0 # via tox pyproject-api==1.11.0 # via tox -python-discovery==1.5.2 +python-discovery==1.6.0 # via # tox # virtualenv tomli-w==1.2.0 # via tox -tox==4.60.0 +tox==4.60.1 # via -r requirements/ci.in typing-extensions==4.16.0 # via tox -virtualenv==21.7.4 +virtualenv==21.7.7 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index c47d8e064..28854da8c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/quality.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/quality.txt + # pydantic asgiref==3.12.1 # via # -r requirements/quality.txt @@ -29,7 +33,7 @@ billiard==4.2.4 # via # -r requirements/quality.txt # celery -build==1.5.0 +build==1.6.0 # via # -r requirements/pip-tools.txt # pip-tools @@ -54,7 +58,7 @@ charset-normalizer==3.5.1 # via # -r requirements/quality.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt @@ -96,7 +100,7 @@ coverage[toml]==7.15.4 # via # -r requirements/quality.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/quality.txt # pyjwt @@ -166,11 +170,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/quality.txt dnspython==2.8.0 # via # -r requirements/quality.txt + # email-validator # pymongo docutils==0.23 # via @@ -206,6 +211,10 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/quality.txt +email-validator==2.3.0 + # via + # -r requirements/quality.txt + # pydantic fastavro==1.12.2 # via # -r requirements/quality.txt @@ -218,7 +227,9 @@ filelock==3.32.4 # virtualenv freezegun==1.5.5 # via -r requirements/quality.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/quality.txt +grimp==3.16 # via # -r requirements/quality.txt # import-linter @@ -229,8 +240,9 @@ id==1.6.1 idna==3.19 # via # -r requirements/quality.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/quality.txt iniconfig==2.3.0 # via @@ -341,7 +353,7 @@ pillow==12.3.0 # edx-organizations pip-tools==7.6.1 # via -r requirements/pip-tools.txt -platformdirs==4.11.3 +platformdirs==4.11.5 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -372,6 +384,12 @@ pycparser==3.0 # via # -r requirements/quality.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/quality.txt +pydantic-core==2.46.5 + # via + # -r requirements/quality.txt + # pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.21.0 @@ -437,7 +455,7 @@ python-dateutil==2.9.0.post0 # -r requirements/quality.txt # celery # freezegun -python-discovery==1.5.2 +python-discovery==1.6.0 # via # -r requirements/ci.txt # tox @@ -451,7 +469,7 @@ pyyaml==6.0.3 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==45.0 +readme-renderer==46.0 # via # -r requirements/quality.txt # twine @@ -518,7 +536,7 @@ tomlkit==0.15.1 # -r requirements/quality.txt # edx-lint # pylint -tox==4.60.0 +tox==4.60.1 # via -r requirements/ci.txt twine==7.0.0 # via -r requirements/quality.txt @@ -537,7 +555,14 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core # tox + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/quality.txt + # pydantic tzdata==2026.3 # via # -r requirements/quality.txt @@ -558,11 +583,11 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==21.7.4 +virtualenv==21.7.7 # via # -r requirements/ci.txt # tox -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/quality.txt # prompt-toolkit diff --git a/requirements/doc.txt b/requirements/doc.txt index 99e4bd15c..f3b2f4a67 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,6 +12,10 @@ amqp==5.3.1 # via # -r requirements/test.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/test.txt + # pydantic asgiref==3.12.1 # via # -r requirements/test.txt @@ -49,7 +53,7 @@ charset-normalizer==3.5.1 # via # -r requirements/test.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/test.txt # celery @@ -77,7 +81,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/test.txt # pyjwt @@ -134,11 +138,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.txt dnspython==2.8.0 # via # -r requirements/test.txt + # email-validator # pymongo doc8==2.0.0 # via -r requirements/doc.in @@ -175,23 +180,30 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/test.txt +email-validator==2.3.0 + # via + # -r requirements/test.txt + # pydantic fastavro==1.12.2 # via # -r requirements/test.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/test.txt +grimp==3.16 # via # -r requirements/test.txt # import-linter idna==3.19 # via # -r requirements/test.txt + # email-validator # requests -imagesize==2.0.0 +imagesize==2.0.1 # via sphinx -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.txt iniconfig==2.3.0 # via @@ -270,6 +282,12 @@ pycparser==3.0 # via # -r requirements/test.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/test.txt +pydantic-core==2.46.5 + # via + # -r requirements/test.txt + # pydantic pydata-sphinx-theme==0.20.0 # via sphinx-book-theme pygments==2.21.0 @@ -317,7 +335,7 @@ pyyaml==6.0.3 # via # -r requirements/test.txt # code-annotations -readme-renderer==45.0 +readme-renderer==46.0 # via -r requirements/doc.in requests==2.34.2 # via @@ -403,6 +421,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/test.txt + # pydantic tzdata==2026.3 # via # -r requirements/test.txt @@ -421,7 +446,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 9e327bfba..008bd5b92 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,9 +4,9 @@ # # make upgrade # -build==1.5.0 +build==1.6.0 # via pip-tools -click==8.4.2 +click==8.5.0 # via pip-tools packaging==26.3 # via diff --git a/requirements/quality.txt b/requirements/quality.txt index ad1da6fd9..7e2f9d426 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/test.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/test.txt + # pydantic asgiref==3.12.1 # via # -r requirements/test.txt @@ -43,7 +47,7 @@ charset-normalizer==3.5.1 # via # -r requirements/test.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/test.txt # celery @@ -77,7 +81,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.txt # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/test.txt # pyjwt @@ -136,11 +140,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.txt dnspython==2.8.0 # via # -r requirements/test.txt + # email-validator # pymongo docutils==0.23 # via readme-renderer @@ -172,13 +177,19 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/test.txt +email-validator==2.3.0 + # via + # -r requirements/test.txt + # pydantic fastavro==1.12.2 # via # -r requirements/test.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.txt -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/test.txt +grimp==3.16 # via # -r requirements/test.txt # import-linter @@ -187,8 +198,9 @@ id==1.6.1 idna==3.19 # via # -r requirements/test.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.txt iniconfig==2.3.0 # via @@ -270,7 +282,7 @@ pillow==12.3.0 # via # -r requirements/test.txt # edx-organizations -platformdirs==4.11.3 +platformdirs==4.11.5 # via pylint pluggy==1.6.0 # via @@ -291,6 +303,12 @@ pycparser==3.0 # via # -r requirements/test.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/test.txt +pydantic-core==2.46.5 + # via + # -r requirements/test.txt + # pydantic pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.21.0 @@ -348,7 +366,7 @@ pyyaml==6.0.3 # via # -r requirements/test.txt # code-annotations -readme-renderer==45.0 +readme-renderer==46.0 # via twine requests==2.34.2 # via @@ -417,6 +435,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/test.txt + # pydantic tzdata==2026.3 # via # -r requirements/test.txt @@ -437,7 +462,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 6a1fe812f..33ea67cc8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,6 +8,10 @@ amqp==5.3.1 # via # -r requirements/base.txt # kombu +annotated-types==0.8.0 + # via + # -r requirements/base.txt + # pydantic asgiref==3.12.1 # via # -r requirements/base.txt @@ -37,7 +41,7 @@ charset-normalizer==3.5.1 # via # -r requirements/base.txt # requests -click==8.4.2 +click==8.5.0 # via # -r requirements/base.txt # celery @@ -65,7 +69,7 @@ coverage[toml]==7.15.4 # via # -r requirements/test.in # pytest-cov -cryptography==50.0.0 +cryptography==50.0.1 # via # -r requirements/base.txt # pyjwt @@ -118,11 +122,12 @@ djangorestframework==3.18.0 # drf-jwt # edx-drf-extensions # edx-organizations -djangorestframework-stubs==3.18.0 +djangorestframework-stubs==3.18.1 # via -r requirements/test.in dnspython==2.8.0 # via # -r requirements/base.txt + # email-validator # pymongo drf-jwt==1.19.2 # via @@ -150,19 +155,26 @@ edx-opaque-keys[django]==4.0.0 # openedx-events edx-organizations==9.0.0 # via -r requirements/base.txt +email-validator==2.3.0 + # via + # -r requirements/base.txt + # pydantic fastavro==1.12.2 # via # -r requirements/base.txt # openedx-events freezegun==1.5.5 # via -r requirements/test.in -grimp==3.15 +fsspec==2026.7.0 + # via -r requirements/base.txt +grimp==3.16 # via import-linter idna==3.19 # via # -r requirements/base.txt + # email-validator # requests -import-linter==2.13 +import-linter==2.14 # via -r requirements/test.in iniconfig==2.3.0 # via pytest @@ -217,6 +229,12 @@ pycparser==3.0 # via # -r requirements/base.txt # cffi +pydantic[email]==2.13.5 + # via -r requirements/base.txt +pydantic-core==2.46.5 + # via + # -r requirements/base.txt + # pydantic pygments==2.21.0 # via # pytest @@ -297,6 +315,13 @@ typing-extensions==4.16.0 # edx-opaque-keys # import-linter # mypy + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # -r requirements/base.txt + # pydantic tzdata==2026.3 # via # -r requirements/base.txt @@ -315,7 +340,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.8.2 +wcwidth==0.8.3 # via # -r requirements/base.txt # prompt-toolkit diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 8f880aa7b..11f2217e6 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -1,42 +1,105 @@ """ Backup Restore API + +Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage + +Extract -> Validate -> Load + + +(FS + root) -> UnvalidatedLearningPackage -> ValidatedLearningPackageInput + """ -import zipfile +from datetime import datetime, timezone +import attrs from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user +from django.db.transaction import atomic -from ..publishing.api import get_learning_package_by_ref -from .zipper import LearningPackageUnzipper, LearningPackageZipper +from ..publishing import api as publishing_api +from . import archive, loading, payload, validation -# The public API that will be re-exported by openedx_content.api -# is listed in the __all__ entries below. Internal helper functions that are -# private to this module should start with an underscore. If a function does not -# start with an underscore AND it is not in __all__, that function is considered -# to be callable only by other applets in the openedx_content package. -__all__ = [ - "create_zip_file", - "load_learning_package", -] +from .zipper import LearningPackageZipper, generate_staged_package_ref + + +@attrs.define(frozen=True) +class ImportResult: + entities_created: int # Should this be a list of entity refs instead? + + +def load_learning_package( + path_str: str, + user: UserType, + package_ref: str | None = None, +) -> dict: + """ + Loads a learning package from a zip file at the given path. + + Restores the learning package and its contents to the database. + + The overall pipeline looks like this: + Archive location (Path) → + FileSystem (fsspec) → + UnvalidatedLearningPackageInput → + ValidatedLearningPackageInput → + LearningPackage + + TODO: Returns a dictionary with the status of the operation and any errors encountered. + + Loads a learning package from a zip file at the given path. + Restores the learning package and its contents to the database. + Returns a dictionary with the status of the operation and any errors encountered. + """ + fs = archive.read_fs_for_path(path_str) + unvalidated_input = payload.extract_unvalidated_learning_package(fs) + + # TODO: need to be able to exit early here if errors make the rest of this + # pointless. The Loader class currently knows how to make output that we can + # send up to platform, but maybe that knowledge should be in this module + # instead? + # if unvalidated_input.errors: + validated_input = validation.validate(unvalidated_input) + + if package_ref is None: + package_ref = generate_staged_package_ref( + validated_input.data.learning_package.key, user, + ) + + loader = loading.Loader(validated_input) + now = datetime.now(tz=timezone.utc) + with atomic(savepoint=False): + learning_package = publishing_api.create_learning_package( + package_ref, "Temp Title", created=now + ) + load_target = loading.Loader.Target(learning_package, user, now) + result = loader.load_into(load_target) + + return result + + +def pretty_print(obj): + from pydantic import TypeAdapter + from typing import Any + from rich import print_json + + print_json(TypeAdapter(Any).dump_json(obj, indent=2).decode("utf8")) + + +### This was pre-existing: def create_zip_file( - package_ref: str, path: str, user: UserType | None = None, origin_server: str | None = None + lp_key: str, + path: str, + user: UserType | None = None, + origin_server: str | None = None, ) -> None: """ Creates a dump zip file for the given learning package key at the given path. The zip file contains a TOML representation of the learning package and its contents. + This is used by lp_dump. + Can throw a NotFoundError at get_learning_package_by_ref """ - learning_package = get_learning_package_by_ref(package_ref) + learning_package = publishing_api.get_learning_package_by_ref(lp_key) LearningPackageZipper(learning_package, user, origin_server).create_zip(path) - - -def load_learning_package(path: str, package_ref: str | None = None, user: UserType | None = None) -> dict: - """ - Loads a learning package from a zip file at the given path. - Restores the learning package and its contents to the database. - Returns a dictionary with the status of the operation and any errors encountered. - """ - with zipfile.ZipFile(path, "r") as zipf: - return LearningPackageUnzipper(zipf, package_ref, user).load() diff --git a/src/openedx_content/applets/backup_restore/archive.py b/src/openedx_content/applets/backup_restore/archive.py new file mode 100644 index 000000000..418739a13 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/archive.py @@ -0,0 +1,32 @@ +""" +This module exists to abstract away the container archive format. To being with, +we are supporting Zip files and simple directories (useful for testing). +""" +from pathlib import Path + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem +from fsspec import AbstractFileSystem + +def read_fs_for_path(path_str: str) -> AbstractFileSystem: + """ + If the path_str passed in is a directory, we treat that as the root of the + archive to be restored. Otherwise, we assume you're passing a Zip file. + + For future consideration: Using LibArchiveFileSystem would allow us to + support tar.gz, zip, 7z, and a bunch of other archiving formats in read-only + mode. I'm not doing it now because I'm not clear on whether the reliance on + libarchive makes things problematic, I don't understand the performance + implications, and I don't want to open the door on "supported archive + formats" to include everything under the sun. But it's an intriguing option + to consider. + + TODO: Can we force read-only mode on these file systems? + """ + path = Path(path_str) + if path.is_dir(): + return DirFileSystem(path) + elif path.is_file() and path.suffix.lower() == ".zip": + return ZipFileSystem(path) + + raise ValueError(f"Could not load path {path_str}") diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py new file mode 100644 index 000000000..984743844 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -0,0 +1,326 @@ +""" +Logic for taking the logical schema model for a Learning Package and loading it +into the database. +""" +import mimetypes +import os.path + +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from functools import cache, partial + +from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user + +from ..components import api as components_api +from ..containers import api as containers_api +from ..collections import api as collections_api +from ..media import api as media_api +from ..publishing import api as publishing_api +from ..publishing.models import LearningPackage +from ..sections.models import Section +from ..subsections.models import Subsection +from ..units.models import Unit + +from .schema import ( + SectionInputData, + SubsectionInputData, + UnitInputData, +) +from .validation import ValidatedLearningPackageInput +from .zipper import RestoreResult, RestoreLearningPackageData, BackupMetadata + + +class Loader: + """ + Loads the validated input into a Learning Package in the database. + + This class does not understand the specifics of the archive file format. It + only needs the ValidatedLearningPackageInput. + """ + + @dataclass(frozen=True) + class Target: + learning_package: LearningPackage + user: UserType + loaded_at: datetime + + def __init__(self, validated_input: ValidatedLearningPackageInput): + self.validated_input = validated_input + self.component_inputs = {} + self.section_inputs = {} + self.subsection_inputs = {} + self.unit_inputs = {} + + entities = validated_input.data.entities + + # Split our entities into separate dicts for convenience. + for entity_ref, entity_input in sorted(entities.items()): + match entity_input.container: + case SectionInputData(): + self.section_inputs[entity_ref] = entity_input + case SubsectionInputData(): + self.subsection_inputs[entity_ref] = entity_input + case UnitInputData(): + self.unit_inputs[entity_ref] = entity_input + case None: + # For the moment, if it's not a Container, it's a Component + self.component_inputs[entity_ref] = entity_input + + def load_into(self, target: Target): + """ + This method intentionally takes a target (LearningPackage, User, + Datetime) instead of putting that information into Loader object state. + My hope is that this pattern will make it easier to adapt into handling + incremental imports where we have to test the same input being imported + into multiple Learning Packages with existing state. + """ + bulk_change_context_for_time = partial( + publishing_api.bulk_draft_changes_for, + target.learning_package.id, + changed_by=target.user.id, + ) + + # DraftChangeLog 1: Add all the PublishableEntities and their versions, + # and set their versions to prepare for for publishing. + with bulk_change_context_for_time(changed_at=target.loaded_at): + loaded_components = self.load_components_into(target) + loaded_entities = self.load_containers_into(target, loaded_components) + self.set_draft_versions(target, for_publishing=True) + + publishing_api.publish_all_drafts( + target.learning_package.id, + published_at=target.loaded_at, + published_by=target.user.id, + message="Restore from backup.", + ) + + # DraftChangeLog 2: Set all PublishableEntities to their proper draft. + # At this point, all versions have been loaded, and the correct versions + # have been published, but the current draft version might be wrong. + # + # The history display will want draft changes to be slightly after the + # published log entry. + changed_at = target.loaded_at + timedelta(seconds=1) + with bulk_change_context_for_time(changed_at=changed_at): + self.set_draft_versions(target, for_publishing=False) + + # Collections are added at the end, in case publishing of contents would + # cause more thrashing w.r.t. search indexing. + self.load_collections_into(target, loaded_entities) + + return self.build_restore_result(target) + + def build_restore_result(self, target: Target): + """ + This is for compatibility with what we're already sending to the frontend. + + TODO: We should return something more structured for our API and let the + calling api.py handle the translation into what the REST API expects. + """ + validated_data = self.validated_input.data + + # Fix this with better parsing later. + _lib, org, slug = validated_data.learning_package.key.split(":") + + loaded_entities = publishing_api.get_publishable_entities(target.learning_package.id) + + result = RestoreResult( + status="success", + log_file_error=None, + lp_restored_data=RestoreLearningPackageData( + id=target.learning_package.id, + key=target.learning_package.key, + archive_lp_key=validated_data.learning_package.key, + archive_org_key=org, + archive_slug=slug, + title=target.learning_package.title, + num_containers=loaded_entities.filter(container__isnull=False).count(), + num_sections=loaded_entities.filter(container__section__isnull=False).count(), + num_subsections=loaded_entities.filter(container__subsection__isnull=False).count(), + num_units=loaded_entities.filter(container__unit__isnull=False).count(), + num_components=loaded_entities.filter(component__isnull=False).count(), + num_collections=collections_api.get_collections(target.learning_package.id).count(), + ), + backup_metadata=BackupMetadata( + format_version=validated_data.meta.format_version, + created_by=validated_data.meta.created_by, + created_by_email=validated_data.meta.created_by, + created_at=validated_data.meta.created_at, + original_server=validated_data.meta.origin_server, + ), + ) + return asdict(result) + + def load_components_into(self, target: Target): + """ """ + + @cache # inner fn, so won't persist across calls to load_components_into + def _get_component_type(namespace: str, name: str): + return components_api.get_or_create_component_type(namespace, name) + + @cache # inner fn, so won't persist across calls to load_components_into + def _get_media_type(mime_type: str): + return media_api.get_or_create_media_type(mime_type) + + mapping = {} + for entity_ref, entity_input in self.component_inputs.items(): + namespace, name, component_code = entity_ref.split(":") + component_type = _get_component_type(namespace, name) + component = components_api.create_component( + target.learning_package.id, + component_type=component_type, + local_key=component_code, + created=target.loaded_at, + created_by=target.user.id, + ) + # TODO: Validate missing children + sorted_version_inputs = sorted( + entity_input.versions, key=lambda v: v.version_num + ) + for version_input in sorted_version_inputs: + media_to_replace = {} + for path, text_val in version_input.component.media.items(): + filename = os.path.basename(path) + if filename == "block.xml": + media_type = _get_media_type( + f"application/vnd.openedx.xblock.v1.{component_type.name}+xml" + ) + else: + media_type_str, _encoding = mimetypes.guess_type(filename) + media_type_str = media_type_str or "application/octet-stream" + media_type = _get_media_type(media_type_str) + + # TODO: Adopt data-urls for this. + if path.startswith('static/'): + # This is where we could add base64 encoded versions + # right now, we just use fs:/path/to/file + _resource_type, filepath = text_val.split(":") + new_media = media_api.get_or_create_file_media( + target.learning_package.id, + media_type.id, + data=self.validated_input.fs.read_bytes(filepath), + created=target.loaded_at, + ) + else: + new_media = media_api.get_or_create_text_media( + target.learning_package.id, + media_type.id, + text=text_val, + created=target.loaded_at, + ) + + media_to_replace[path] = new_media.id + + # TODO: Modify create_next_component_version to take a Component + # as an option, to save the needless fetches. + components_api.create_next_component_version( + component.pk, + title=version_input.title, + media_to_replace=media_to_replace, + created=target.loaded_at, + created_by=target.user.id, + force_version_num=version_input.version_num, + ) + mapping[entity_ref] = component + + return mapping + + def load_containers_into(self, target: Target, component_mapping: dict): + + # Ordering matters, since we want to build the references bottom-up. + container_types_to_inputs = { + Unit: self.unit_inputs, + Subsection: self.subsection_inputs, + Section: self.section_inputs, + } + mapping = component_mapping.copy() + for container_type, container_inputs in container_types_to_inputs.items(): + for entity_ref, entity_input in container_inputs.items(): + container = containers_api.create_container( + target.learning_package.id, + entity_ref, + created=target.loaded_at, + created_by=target.user.id, + container_cls=container_type, + ) + + # TODO: Validate missing children + sorted_version_inputs = sorted( + entity_input.versions, key=lambda v: v.version_num + ) + for version_input in sorted_version_inputs: + containers_api.create_next_container_version( + container, + title=version_input.title, + entities=[ + mapping[child_ref] + for child_ref in version_input.container.children + ], + created=target.loaded_at, + created_by=target.user.id, + force_version_num=version_input.version_num, + ) + + mapping[entity_ref] = container + + return mapping + + def load_collections_into(self, target: Target, loaded_entities): + for collection_input in self.validated_input.data.collections: + collections_api.create_collection( + target.learning_package.id, + key=collection_input.key, + title=collection_input.title, + created_by=target.user.id, + description=collection_input.description, + ) + loaded_entity_refs = [ + ref for ref in collection_input.entities if ref in loaded_entities + ] + entities = publishing_api.get_publishable_entities( + target.learning_package.id + ).filter(key__in=loaded_entity_refs) + + collections_api.add_to_collection( + target.learning_package.id, + key=collection_input.key, + entities_qset=entities, + ) + + def set_draft_versions(self, target: Target, for_publishing: bool): + entity_inputs = self.validated_input.data.entities + + saved_entities = publishing_api.get_publishable_entities( + target.learning_package.id + ) + for saved_entity in saved_entities: + saved_draft_version = publishing_api.get_draft_version(saved_entity) + input_entity = entity_inputs[saved_entity.key] + + if for_publishing: + input_version_num = input_entity.published.version_num + else: + input_version_num = input_entity.draft.version_num + + # The version we want to set is already the current draft, which + # means there's nothing to do. + if ( + saved_draft_version + and saved_draft_version.version_num == input_version_num + ): + continue + + if input_version_num is None: + version_id_to_set = None + else: + version_model_to_publish = saved_entity.versions.get( + version_num=input_version_num + ) + version_id_to_set = version_model_to_publish.id + + publishing_api.set_draft_version( + saved_entity.id, + version_id_to_set, + set_at=target.loaded_at, + set_by=target.user.id, + ) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py new file mode 100644 index 000000000..b6e5d1135 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -0,0 +1,511 @@ +""" +This module works with the actual files in our backup archive. It is agnostic to +the archive container format that the files are bundled in, e.g. a local file +system directory, a zip file archive, or something more exotic down the line. + +Some high level considerations for this module: + +1. The error checking is for the file format itself, i.e. extracting values + from the TOML files and static assets and assembling them for validation. + In some cases, this means we do have to look for particular fields to handle +""" + +from __future__ import annotations +from numbers import Number +import os.path # fsspec doesn't work well with Path objects. +import tomllib + +import attrs +from fsspec import AbstractFileSystem + +ROOT_PACKAGE_PATH = "package.toml" + + +@attrs.define(frozen=True) +class UnvalidatedLearningPackageInput: + raw_data: dict + errors: list[ExtractionError] + fs: AbstractFileSystem + + # Mapping of entity refs to the paths where we found them. + entity_path_mapping: dict[str, str] + + +class ExtractionError(Exception): + """ + Any error during the extraction process. + + At the moment, any error is fatal. The point of the different errors is to + provide useful debug logging and to let us write tests that look for + specific errors. + """ + + def __init__(self, message, path=None): + super().__init__(message) + self.message = message + self.path = path + + def __str__(self): + return f"{self.path}: {self.message}" + + +class InvalidTOMLError(ExtractionError): + def __init__(self, file_description, details, path): + message = f"Cannot decode TOML for {file_description}: {details}" + super().__init__(message, path=path) + + +class TableNotFoundError(ExtractionError): + def __init__(self, file_description, table, path): + self.table = table + message = f"Table [{table}] not found in {file_description}." + super().__init__(message, path=path) + + +class FieldsNotInTable(ExtractionError): + def __init__(self, file_description, fields, path): + self.fields = sorted(fields) + message = f"{file_description} has fields not in a table: {', '.join(fields)}" + super().__init__(message, path=path) + + +class FieldMissing(ExtractionError): + def __init__(self, file_description, table, missing_field, path): + self.table = table + self.missing_field = missing_field + message = ( + f'{file_description} is missing required field "{missing_field}" ' + f"from table [{table}]" + ) + super().__init__(message, path=path) + + +class FileNotFoundError(ExtractionError): + def __init__(self, file_description, path): + message = f"{file_description} file not found at expected path" + super().__init__(message, path=path) + + +class DuplicateFoundError(ExtractionError): + def __init__(self, description, original_path, path): + self.original_path = original_path + message = f"{description} already defined in {original_path}" + super().__init__(message, path=path) + + +class UnsupportedFormatError(ExtractionError): + pass + + +class PayloadExtractor: + """ + Extracts files from a file system and generates unvalidated input. + """ + + def __init__(self, fs: AbstractFileSystem): + self.fs = fs + + if self.fs.exists("package.toml"): + self.root = "" + elif len(fs.ls('.')) == 1: + pass + + + +def extract_unvalidated_learning_package( + fs: AbstractFileSystem, +) -> UnvalidatedLearningPackageInput: + """ + Extract the raw, unvalidated Learning Package metadata. + + We scan through the archive and compile a Python dictionary that can be + validated against CompletePackageInputData. This mostly involves reading a + bunch of TOML-serialized files and copying their contents, with some minor + data transformations where idiomatic TOML doesn't really give the structure + we want in our final model. + + The purpose of this abstraction is to later allow different ways to assemble + the JSON that we want to do our validation on. By default, this is a bunch + of TOML files, but folks who have specialized authoring needs may prefer a + different set of conventions. For instance, the MIT Disciplinary Experts in + Learning Technology and Applications team prefers to author in a way that + encodes large parts of the hierarchy (Section -> Subsection -> Unit) in a + single file, with pointers to certain Components in different files. + + Errors can happen at this layer, but they are errors related to the + consistency of the archive payload format itself. So errors that need to be + checked here are things like: + + * Missing critical files, like package.toml + * Duplicated entity files, as this is not possible to represent in the + CompletePackageInputData schema. + + Things like missing fields and incorrect field values will be handled at the + validation step which happens after this. In other words, the only things + that are errors here are the things that prevent us from creating a + UnvalidatedLearningPackageInput at all. + """ + # The general philosophy here is to always march on and get as much as + # possible, even if we know the upload is doomed. + unvalidated = {} + errors = [] + + # Root Package Metadata + try: + # This adds the "meta" and "learning_package" keys + unvalidated |= extract_root_package_data(fs, ROOT_PACKAGE_PATH) + except ExtractionError as err: + errors.append(err) + + # PublishableEntities & versions (components, units, sections, subsections) + entities_data, entity_path_mapping, entities_errors = extract_entities_data( + fs, get_entity_file_paths(fs) + ) + unvalidated["entities"] = entities_data + errors.extend(entities_errors) + + # Collections + # TODO: Duplicate collections are a problem too. + collections = [] + for collection_file_path in sorted(fs.glob("collections/*.toml")): + try: + collections.append(extract_collection_data(fs, collection_file_path)) + except ExtractionError as err: + errors.append(err) + unvalidated["collections"] = collections + + return UnvalidatedLearningPackageInput( + raw_data=unvalidated, + errors=errors, + fs=fs, + entity_path_mapping=entity_path_mapping, + ) + + +def extract_root_package_data(fs: AbstractFileSystem, path: str) -> dict: + """ + Extract the "meta" and "learning_package" from the TOML file at path. + + This is a straightforward extraction because we don't have to transform the + actual fields in the data. We expect to see a TOML file that looks something + like this: + + [meta] + format_version = 1 + created_by = "eddy" + created_by_email = "eddy@axim.org" + created_at = 2026-03-11T19:20:20.394360Z + origin_server = "studio.local.openedx.io" + + [learning_package] + title = "Fun Library" + key = "lib:Axim:FunLib" + description = "My very fun library! 🐢" + created = 2026-02-11T16:32:47.524556Z + updated = 2026-02-20T16:32:47.524556Z + + The output should look like: + + { + 'meta': { + 'format_version': 1, + 'created_by': 'eddy', + 'created_by_email': 'eddy@axim.org', + 'created_at': datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), + 'origin_server': 'studio.local.openedx.io' + }, + 'learning_package': { + 'title': 'Fun Library', + 'key': 'lib:Axim:FunLib', + 'description': 'My very fun library! 🐢', + 'created': datetime(2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc), + 'updated': datetime(2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc), + } + } + + We need to return a Python dict that we get from parsing this. Most of this + function is error handling. The error checking at this layer is minimal, and + is mostly focused on making sure that the file exists, is parseable, and has + the two tables we expect it to have. + """ + file_description = "Root Package" + + # Check: Root Package file exists at all. + if not fs.exists(path): + raise FileNotFoundError(file_description, path=path) + + # Check: Is it a valid TOML file? + with fs.open(path, "rb") as package_toml_file: + try: + root_package_dict = tomllib.load(package_toml_file) + except tomllib.TOMLDecodeError as dec_err: + raise InvalidTOMLError( + file_description, details=str(dec_err), path=path + ) from dec_err + + # Check: Don't allow top-level fields outside a [table] + _check_all_fields_in_tables(root_package_dict, file_description, path) + + # Check: The "[meta]" and "[learning_package]" tables are mandatory + if "meta" not in root_package_dict: + raise TableNotFoundError(file_description, table="meta", path=path) + if "learning_package" not in root_package_dict: + raise TableNotFoundError(file_description, table="learning_package", path=path) + + # Check: We only support format_version 1, and don't know what to do with + # anything higher. This leaves us some wiggle-room to declare a 1.x version + # that is backwards compatible, i.e. it will reject 2 and higher, but accept + # 1.1, 1.2, etc. + format_version = root_package_dict["meta"].get("format_version") + if not isinstance(format_version, Number) or format_version >= 2: + raise UnsupportedFormatError( + f"Format version {format_version} is unsupported (only 1 is supported).", + path=path, + ) + + return root_package_dict + + +def get_entity_file_paths(fs: AbstractFileSystem) -> list[str]: + """ + Find all the PublishableEntity TOML file paths in our archive. + + We expect our entity TOML files to be in the entities directory, but we have + two categories right now: + + * Component TOML: entities/xblock.v1/{component_type}/{component_code} + * Container TOML: entities/{entity_ref} + + This function looks for TOML files in entities/ or any of its subdirs. We + only exclude matches inside the component_version data, to make sure that we + don't accidentally match media files in the unlikely event where people have + TOML files as static assets. + """ + paths = [ + path + for path in fs.glob("entities/**/*.toml") + # Filter out TOML files that are in component media, e.g. static assets: + if "/component_versions/" not in path + ] + return sorted(paths) # Make the ordering deterministic. + + +def extract_entities_data(fs: AbstractFileSystem, paths: list[str]): + entities_data = {} + entity_path_mapping = {} + errors = [] + for entity_file_path in paths: + try: + entity_ref, entity_data = extract_entity_data( + fs, entity_file_path, entity_path_mapping + ) + entities_data[entity_ref] = entity_data + entity_path_mapping[entity_ref] = entity_file_path + except ExtractionError as err: + errors.append(err) + + return entities_data, entity_path_mapping, errors + + +def extract_entity_data( + fs: AbstractFileSystem, path: str, entity_path_mapping: dict[str, str] | None = None +) -> tuple[str, dict]: + """ + This extracts raw entity data from an Entity TOML file. + + PublishableEntities can be both Components (XBlock problems, videos, etc.), + as well as Containers like Units, Subsections, and Sections. Some sample + TOML: + + [entity] + can_stand_alone = true + key = "section-9-ac4b9f" + created = 2026-04-08T15:22:12.780012Z + + [entity.draft] + version_num = 2 + + [entity.published] + version_num = 1 + + [entity.container.section] + + # ### Versions + + [[version]] + title = "Section 9" + version_num = 2 + + [version.container] + children = ["week-7-e73782", "subsection-001-e4bbe5"] + + [[version]] + title = "Section 9" + version_num = 1 + + [version.container] + children = ["week-7-e73782"] + + We return a tuple where the first element is the Entity's key + ("section-9-ac4b9f"), and the second is a dict that would look like: + + { + 'can_stand_alone': True, + 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), + 'draft': { + 'version_num': 2 + }, + 'published': { + 'version_num': 1 + }, + 'container': { + 'section': {} + }, + 'versions': [ + { + 'title': 'Section 9', + 'version_num': 2, + 'container': { + 'children': [ + 'week-7-e73782', + 'subsection-001-e4bbe5' + ] + } + }, + { + 'title': 'Section 9', + 'version_num': 1, + 'container': { + 'children': [ + 'week-7-e73782' + ] + } + } + ] + } + + Note some key differences: + + 1. The "entity" table elements have been popped out to the top level. + 2. The "version" list has been renamed to "versions" to feel more natural. + 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back as + part of the tuple. This will become a key/value pair in an "entities" + dict that will hold all publishable entity input data. + """ + file_description = "Entity" + if entity_path_mapping is None: + entity_path_mapping = {} + + # Check: Is it a valid TOML file? + with fs.open(path, "rb") as entity_file: + try: + entity_root_dict = tomllib.load(entity_file) + except tomllib.TOMLDecodeError as dec_err: + raise InvalidTOMLError( + file_description, details=str(dec_err), path=path + ) from dec_err + + # Check: Don't allow top-level fields outside a [table] + _check_all_fields_in_tables(entity_root_dict, file_description, path) + + # Check: Does it define a top level "[entity]" table? Note that this can + # pass if they define a sub-table like "[entity.draft]", since the existence + # of "[entity]" is implicit in that case. If we get that far, rely on + # catching it at the validation step (i.e. after payload extraction). + if "entity" not in entity_root_dict: + raise TableNotFoundError(file_description, "entity", path=path) + + # Check: Does it define an Entity key (i.e. entity_ref)? We need to check + # this now because the dict we have to assemble will use these as keys. + entity = entity_root_dict["entity"] + entity_ref = entity.pop("key", None) + if not entity_ref: + raise FieldMissing(file_description, "entity", "key", path) + + # Check: Is it a duplicate of an Entity that has already been defined + # elsewhere in this archive? + if entity_ref in entity_path_mapping: + raise DuplicateFoundError( + f"Entity key {entity_ref}", entity_path_mapping[entity_ref], path + ) + + # Note case difference: we're renaming "version" in the TOML to "versions" + # in the data dict we're assembling. + entity["versions"] = entity_root_dict.pop("version", []) + for version in entity["versions"]: + # Do our best to put together entity version data (and component version + # data), but don't worry about validating the results (that can happen + # during the validation step). + version_num = version.get("version_num") + comp_ver_dir = os.path.join( + os.path.splitext(path)[0], + "component_versions", + f"v{version_num}", + ) + if fs.exists(comp_ver_dir): + version["component"] = {} + media = { + os.path.relpath(path, comp_ver_dir): fs.read_text(path) + for path in fs.glob(f"{comp_ver_dir}/*") + if fs.isfile(path) + } + # Any static files are encoded as pointers. + # TODO: Convert this to data-urls later + for static_file_path in fs.glob(f"{comp_ver_dir}/static/**"): + if fs.isfile(static_file_path): + rel_path = os.path.relpath(static_file_path, comp_ver_dir) + media[rel_path] = f"fs:{static_file_path}" + + version["component"]["media"] = media + + return entity_ref, entity + + +def extract_collection_data(fs: AbstractFileSystem, path: str) -> dict: + file_description = "Collection" + + with fs.open(path, "rb") as collection_toml_file: + try: + collection_root_dict = tomllib.load(collection_toml_file) + except tomllib.TOMLDecodeError as dec_err: + raise InvalidTOMLError(file_description, details=str(dec_err), path=path) + + _check_all_fields_in_tables(collection_root_dict, file_description, path) + if "collection" not in collection_root_dict: + raise TableNotFoundError( + file_description, table="collection", path=path + ) + + collection_data = collection_root_dict["collection"] + collection_data["src_path"] = path + + return collection_data + + +def _check_all_fields_in_tables(data: dict, file_description, path): + """ + Raise an error if fields are declared outside of a table. + + The convention for our TOML files is that keys are always in a table, so if + it's *not* in a table, that's likely an omission/error that might otherwise + be difficult to catch because they'd be "missing" from the place they're + supposed to be in the parsed data structure, but that wouldn't be obvious to + someone editing the files by hand. + """ + fields_outside_of_tables = [ + field + for field, val in data.items() + if not isinstance(val, dict) and not isinstance(val, list) + ] + if fields_outside_of_tables: + raise FieldsNotInTable( + file_description, fields=fields_outside_of_tables, path=path + ) + + +def pretty_print(obj): + from pydantic import TypeAdapter + from typing import Any + from rich import print_json + + print_json(TypeAdapter(Any).dump_json(obj, indent=2).decode("utf8")) diff --git a/src/openedx_content/applets/backup_restore/readme.rst b/src/openedx_content/applets/backup_restore/readme.rst new file mode 100644 index 000000000..687bfea68 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/readme.rst @@ -0,0 +1,30 @@ +Backup/Restore Applet +===================== + +The ``backup_restore`` is responsible for making a backup archive of an existing Learning Package, or creating a new Learning Package from an existing archive. + +Motivation +---------- + + +Intended Use Cases +------------------ + + + +Architecture Guidelines +----------------------- + + + +Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage + +Extract -> Validate -> Load + + +Archive + +We are very intentionally separating the following aspects: + +archive.py + The actual \ No newline at end of file diff --git a/src/openedx_content/applets/backup_restore/schema.py b/src/openedx_content/applets/backup_restore/schema.py new file mode 100644 index 000000000..6bde51697 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/schema.py @@ -0,0 +1,302 @@ +""" +This module defines the schema that we use during the backup/restore process. + +The pydantic models defined in this module are divided into InputData and +OutputData. These are intentionally kept separate and do not inherit from each +other. The InputData classes will be much more permissive, with many optional +fields. The OutputData classes are meant for internal use when generating +exports, and will be stricter. +""" +from __future__ import annotations +from pathlib import Path + +from pydantic import ( + AwareDatetime, + BaseModel, + ConfigDict, + Field, + EmailStr, + StrictStr, + StringConstraints, + field_validator +) +from typing import Annotated, Literal + +# Refs are arbitrary identifiers that we do almost no validation of, and are +# mainly there to assure uniqueness within some namespace. +REF_CONSTRAINTS = StringConstraints( + strict=True, + strip_whitespace=True, +), + +# This is for things like the collection_code, library_code, etc. +CODE_CONSTRAINTS = StringConstraints( + strict=True, + strip_whitespace=True, + # Note that we can't use \Z to indicate the end of line in our regex because + # that's not supported syntax in JavaScript, and pydantic will raise an + # error when trying to generate a JSON Schema. However, the combination of $ + # and strip_whitespace=True means that we're sure that we won't allow any + # trailing newlines. + pattern=r"^[a-zA-Z0-9_.-]+$", +), + + +class InputData(BaseModel): + """ + Base class for all inputs, here to set config defaults. + + InputData classes are frozen, i.e. they should only be initialized once from + the unvalidated input. Allowing gradual mutations makes things much harder + to debug. + + InputData clases are also set to allow parameters that they don't recognize + (extra="allow") for the sake of forwards compatibility. As any given file + format gets iterated on, it will get new attributes. Older installs of the + platform should ignore these new attributes and just load the things that we + know how to handle. The reason we don't set this to "ignore" is because + unrecognized fields could be simple typos of known fields, so we still want + to capture that information so we can potentially display warnings about it. + """ + model_config = ConfigDict(frozen=True, extra="allow") + + +class CompletePackageInputData(InputData): + """ + The contents of the entire Learning Package. + """ + meta: MetaInputData + learning_package: LearningPackageInputData + + # Mapping of entity refs to EntityInputData + entities: dict[Annotated[str, REF_CONSTRAINTS], EntityInputData] + + collections: list[CollectionInput] + + @field_validator('collections', mode='after') + @classmethod + def check_for_duplicate_keys(cls, collections: list[CollectionInput]): + """ + Raise a ValueError if we encounter a duplicate collection entry. + + In the longer term, we may want to be able to remove the duplicate + entries (and other broken entries), while still otherwise allowing the + restore to proceed. But for now, any error kills the restore process. + """ + collection_keys_to_paths = {} + for collection in collections: + if collection.key in collection_keys_to_paths: + originally_defined_collection = collection_keys_to_paths[collection.key] + raise ValueError( + f'Collection "{collection.key}" redefined in ' + f'{collection.src_path} (original in ' + f'{originally_defined_collection.src_path})' + ) + else: + collection_keys_to_paths[collection.key] = collection + + return collections + + +class MetaInputData(InputData): + """ + Input Package Metadata, Version 1 + + This is data about the backup file itself, as opposed to the Learning + Package that it contains: who created this backup, when was it created, etc. + On the input side, the fields here are only here so that we can give useful + preview information when the user is uploading this to a new instance. None + of these values are necessary for creating a new Learning Package—in fact, + none of these can even be trusted, since a malicious actor could manipulate + them to say whatever they wanted. It's just meant as a sanity check to help + assure the user that they're restoring the correct package archive. + + The only truly critical field is ``format_version``, since that will one day + affect input validation rules. + """ + format_version: Literal[1] # Only supported version at the moment + created_by: StrictStr | None = Field(min_length=1) + created_by_email: EmailStr | None + created_at: AwareDatetime | None + origin_server: StrictStr | None + +class LearningPackageInputData(InputData): + """ + High level data for a Learning Package itself (not its contents). + """ + title: StrictStr = Field(min_length=1, default="Untitled Library") + key: Annotated[ + str, + REF_CONSTRAINTS, + Field( + description=( + "This is often a LibraryLocatorV2-formatted string, but can be " + "any arbitrary string at the moment. It must be unique within a" + " given server instance." + ), + examples=[ + "lib:OrgName:LibraryName", + "lib:Axim:IntroPhysics", + "lp-restore:Axim:IntroPhysics:1775752130941", + ], + ), + ] + description: StrictStr | None = Field(default="", max_length=10_000) + created: AwareDatetime | None + updated: AwareDatetime | None + + +class DraftInputData(InputData): + version_num: Annotated[int, Field(gt=0)] | None = None + + +class PublishedInputData(InputData): + version_num: Annotated[int, Field(gt=0)] | None = None + + +class EntityInputData(InputData): + can_stand_alone: bool = True + + # key: str + created: AwareDatetime + + # Weird edge case: If you create something, never publish it, and then do a + # "reset to published state", the resulting export in Ulmo would omit the + # [entity.draft] section entirely, rather than it being an empty dictionary. + draft: DraftInputData = DraftInputData(version_num=None) + published: PublishedInputData = PublishedInputData(version_num=None) + + versions: list[VersionInput] = [] + + # Not all entities are containers, and we may one day have containers that + # this version of the code does not understand. So we have a generic dict + # for unknown containers and None means it's something that is not a + # container. + # + # TODO: Test unknown container type. + container: UnitInputData | SubsectionInputData | SectionInputData | dict | None = None + + +class SectionInputData(InputData): + section: dict = {} + +class SubsectionInputData(InputData): + subsection: dict = {} + +class UnitInputData(InputData): + unit: dict = {} + + +class VersionInput(InputData): + version_num: Annotated[int, Field(gt=0)] + title: str + component: ComponentVersionInput | None = None + container: ContainerVersionInput | None = None + + +class ComponentVersionInput(InputData): + media: dict + + +class ContainerVersionInput(InputData): + children: list[Annotated[str, REF_CONSTRAINTS]] + + +class VersionsInput(InputData): + versions: dict[ + Annotated[int, Field(gt=0)], + VersionInput, + ] + + +class CollectionInput(InputData): + title: StrictStr = Field(min_length=1) + key: Annotated[ + str, + CODE_CONSTRAINTS, + Field( + description=( + "A unique slug-like code field. Must be unique within a given Learning Package." + ), + examples=[ + "difficult-problems", + "practice-exams", + ], + ), + ] + description: StrictStr | None = Field(default="", max_length=10_000) + created: AwareDatetime | None + + # It looks like we weren't actually serializing the modified date. + created: AwareDatetime | None + + # This is the source file where this Collection was defined. This is only + # for being able to create useful error messages. We should never be reading + # from this file directly because the exact format of this file should be + # free to change as needed. That's the responsibilty of the payload.py + # module. + src_path: Path | None + + +##### We're not actually using anything below this line yet. + + + + + + +class PackageConfigOutputData(BaseModel): + """ + Writes the package.toml file when we're writing a backup archive. + """ + meta: MetaOutputData + learning_package: LearningPackageOutputData + + +class PackageConfigInputData(BaseModel): + """ + Reads the package.toml file when we're reading from a backup archive. + """ + meta: MetaInputData + learning_package: LearningPackageInputData + + +class MetaOutputData(BaseModel): + """ + Output Package Metadata + + This is metadata that is written so that people can more easily figure out + where a backup archive came from. + + The "created_by", "created_by_email", and "created_at" fields all refer to + the user who created the backup archive, not the user who created the + Library (Learning Package). + """ + format_version: Literal[1] + created_by: StrictStr = Field(min_length=1) + created_by_email: EmailStr + created_at: AwareDatetime + origin_server: StrictStr + + + + + +class LearningPackageOutputData(BaseModel): + """ + High level data for a Learning Package. + """ + title: StrictStr = Field(min_length=1) + key: StrictStr = Field( + pattern=r"^lib:[\w\-.]+:[\w\-.]+$", + description="This is a LibraryLocatorV2", + examples=[ + "lib:OrgName:LibraryName", + "lib:Axim:IntroPhysics", + ] + ) + description: StrictStr + created: AwareDatetime + updated: AwareDatetime + + diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py new file mode 100644 index 000000000..c86063197 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -0,0 +1,39 @@ +""" +This is an archive-agnostic validation of the data models. I might actually just +move this to api.py, since most of this work will be done in schema.py +""" +import attrs + +from pydantic_core import InitErrorDetails +from fsspec import AbstractFileSystem + +from .schema import CompletePackageInputData +from .payload import UnvalidatedLearningPackageInput + +@attrs.define(frozen=True) +class ValidatedLearningPackageInput: + data: CompletePackageInputData | None # None if it's too broken + + fs: AbstractFileSystem + + # All these names are terrible. + + # These are the errors that mean this is actually malformed, i.e. JSON + # Schema level validation. + structural_errors: list[InitErrorDetails] + + deeper_errors: list # This is stuff we have to dig deeper for, e.g. missing parent-child relationship + +def validate( + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> ValidatedLearningPackageInput: + """ """ + validated = CompletePackageInputData.model_validate(unvalidated_lp.raw_data) + # pretty_print(validated) + + return ValidatedLearningPackageInput( + data=validated, + fs=unvalidated_lp.fs, + structural_errors=[], + deeper_errors=[], + ) \ No newline at end of file diff --git a/src/openedx_content/management/commands/encode.py b/src/openedx_content/management/commands/encode.py new file mode 100644 index 000000000..84171c9fb --- /dev/null +++ b/src/openedx_content/management/commands/encode.py @@ -0,0 +1,98 @@ +from datetime import datetime, timezone +import logging + +from django.core.management import CommandError +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + +import json + + +from pydantic import BaseModel, Field +from pydantic.config import ConfigDict +from pydantic.json_schema import models_json_schema + +from typing import Annotated, Optional + + +class EntityVersion(BaseModel): + version_num: int + title: str + +class VersionRef(BaseModel): + version_num: Optional[int] = None + +class Entity(BaseModel): + can_stand_alone: bool + key: str + created: datetime + draft: VersionRef + published: VersionRef + versions: list[EntityVersion] + +class EntityRoot(BaseModel): + entity: Entity + +from openedx_content.applets.backup_restore.schema import ( + LearningPackageOutputData, PackageConfigOutputData, MetaOutputData +) + +class Command(BaseCommand): + """ + Django management command to export a learning package to a zip file. + """ + help = 'Export a learning package to a zip file.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + now = datetime.now(tz=timezone.utc) + config = PackageConfigOutputData( + meta=MetaOutputData( + format_version=1, + created_by="dave", + created_by_email="dave@axim.org", + created_at=now, + ), + learning_package=LearningPackageOutputData( + title="Fun Library", + key="lib:Axim:FunLib", + description="", + created=now, + updated=now, + origin_server="studio.local.openedx.io:8001", + ) + ) + toml_output = tomli_w.dumps(config.model_dump(exclude_defaults=False)) + print(toml_output) + + print(json.dumps(PackageConfigOutputData.model_json_schema(), indent=2)) + + + def handle_old(self, *args, **options): + e = Entity( + can_stand_alone=True, + key="xblock.v1:html:hi-there-9d01929cda81", + created=datetime.now(tz=timezone.utc), + draft=VersionRef(version_num=3), + published=VersionRef(version_num=None), + versions = [ + EntityVersion( + version_num=x, + title=f"Title {x}", + ) + for x in range(10) + ], + ) + base = EntityRoot( + entity=e, + ) + + #toml_output = tomli_w.dumps(base.model_dump(exclude_defaults=True)) + #print(toml_output) + + + print(json.dumps(EntityRoot.model_json_schema(), indent=2)) + diff --git a/src/openedx_content/management/commands/lp_load2.py b/src/openedx_content/management/commands/lp_load2.py new file mode 100644 index 000000000..f23592b09 --- /dev/null +++ b/src/openedx_content/management/commands/lp_load2.py @@ -0,0 +1,67 @@ +""" +Django management commands to handle restore learning packages (WIP) +""" +import logging +import time + +from django.contrib.auth import get_user_model +from django.core.management import CommandError +from django.core.management.base import BaseCommand + +from openedx_content.applets.backup_restore.api import load_learning_package + +logger = logging.getLogger(__name__) + +User = get_user_model() + + + +class Command(BaseCommand): + """ + Django management command to load a Learning Package. + """ + help = 'Load a learning package from a zip file.' + + def add_arguments(self, parser): + parser.add_argument('path', type=str, help='The path of the directory or file to load from.') + parser.add_argument('package_ref', type=str, help="Learning Package Ref: often a v2 library key.") + parser.add_argument('username', type=str, help='The username of the user performing the load operation.') + + + def handle(self, *args, **options): + path = options['path'] + package_ref = options['package_ref'] + username = options['username'] + + user = User.objects.get(username=username) + + load_learning_package(path, user=user, package_ref=package_ref) + + return 0 + if not path.lower().endswith(".zip"): + raise CommandError("Input file name must end with .zip") + try: + start_time = time.time() + # Get the user performing the operation + user = User.objects.get(username=username) + + result = load_learning_package(path, user=user) + duration = time.time() - start_time + if result["status"] == "error": + message = "Errors encountered during restore:\n" + log_buffer = result.get("log_file_error") + if log_buffer: + message += log_buffer.getvalue() + raise CommandError(message) + message = f'{path} loaded successfully (duration: {duration:.2f} seconds)' + self.stdout.write(self.style.SUCCESS(message)) + except FileNotFoundError as exc: + message = f"Learning package file {path} not found: {exc}" + raise CommandError(message) from exc + except Exception as e: + message = f"Failed to load '{path}': {e}" + logger.exception( + "Failed to load zip file %s ", + path, + ) + raise CommandError(message) from e diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml new file mode 100644 index 000000000..6537aa087 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/broken.toml @@ -0,0 +1,2 @@ +[entity] +key = " diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_1.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/dupe_2.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml new file mode 100644 index 000000000..ffee075be --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_key.toml @@ -0,0 +1,10 @@ +# No [entity] key +[entity] +can_stand_alone = true +created = 2025-10-31T16:42:04.158245Z + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml new file mode 100644 index 000000000..9451c4173 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_entity_table.toml @@ -0,0 +1,11 @@ +# Typo: [entitty] instead of [entity] +[entitty] +can_stand_alone = true +key = "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" +created = 2025-10-31T16:42:04.158245Z + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml new file mode 100644 index 000000000..39ff085ff --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_container.toml @@ -0,0 +1,29 @@ +# Pretty normal Section with a couple of child Subsections +[entity] +can_stand_alone = true +key = "section-9-ac4b9f" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 2 + +[entity.published] +version_num = 1 + +[entity.container.section] + +# ### Versions + +[[version]] +title = "Section 9" +version_num = 2 + +[version.container] +children = ["week-7-e73782", "subsection-001-e4bbe5"] + +[[version]] +title = "Section 9" +version_num = 1 + +[version.container] +children = ["week-7-e73782"] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml new file mode 100644 index 000000000..09db43a4a --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[meta] +format_version = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml new file mode 100644 index 000000000..2b05c95ab --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/fields_not_in_table.toml @@ -0,0 +1,15 @@ +# The problem here is that created_by and formt_version are not in a table, and +# we don't allow that. +created_by = "eddy" +format_version = 1 + +[meta] +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml new file mode 100644 index 000000000..4874bb574 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/minimal.toml @@ -0,0 +1,9 @@ +# This is the absolute minimum necessary to pass the payload extraction step. +# It doesn't matter that required fields are missing --> that's handled in +# validation. Payload extraction just requires that the tables we expect exist. +# +# Ordering is also irrelevant for this file. +[learning_package] + +[meta] +format_version = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml new file mode 100644 index 000000000..03e087cf7 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_learning_package.toml @@ -0,0 +1,3 @@ +# This errors because the [learning_package] section is missing. +[meta] +format_version = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml new file mode 100644 index 000000000..95c65a9dd --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/missing_meta.toml @@ -0,0 +1,4 @@ +# This errors because the [meta] section is missing. + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml new file mode 100644 index 000000000..5ff8122fe --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/normal_ulmo_v1.toml @@ -0,0 +1,15 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 +created_by = "eddy" +created_by_email = "eddy@axim.org" +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml new file mode 100644 index 000000000..36d1161ea --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unknown_table.toml @@ -0,0 +1,10 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 + +[learning_package] +title = "Fun Library" + +[unknown] +new_field = "Aloha!" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml new file mode 100644 index 000000000..bbb1d49e2 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_1_1.toml @@ -0,0 +1,7 @@ +# We're going to accept format_version 1.1 as a hedge against backwards +# compatible additions to this format (we may never use this). +[meta] +format_version = 1.1 + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml new file mode 100644 index 000000000..dc2227b41 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_2.toml @@ -0,0 +1,6 @@ +# We don't support format_version > 1 +[meta] +format_version = 2 + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml new file mode 100644 index 000000000..8a9df5dc2 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_b.toml @@ -0,0 +1,6 @@ +# format_version needs to be an integer +[meta] +format_version = "b" + +[learning_package] +title = "Fun Library" diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py new file mode 100644 index 000000000..71c80fefb --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -0,0 +1,229 @@ +""" +IMPORTANT: If you are adding new fields/behaviors, they should take the form of +*new* tests on new test data files, and not modifications to existing ones. +Please be very cautious about whether you are breaking backwards compatibility. + +This module tests our ability to extract data from the backup archive TOML files +and resources, and assemble them into a combined document that represents the +entire LearningPackage, and is encapsulated in UnvalidatedLearningPackageInput. +Most of these test functions that examine individual files. The functions in +payload.py were designed to mostly accept an AbstractFileSystem and path as +arguments, so it should be possible to do simple test calls on TOML files and +dirs without having to mock anything. + +These tests are strictly for the payload module, and therefore don't need Django +to run. +""" + +from datetime import datetime, timezone +from pathlib import Path +from unittest import TestCase, skip + +from fsspec.implementations.dirfs import DirFileSystem + +from openedx_content.applets.backup_restore import payload + + +TEST_DATA_ROOT = Path(__file__).parent / "payload_test_data" + + +class ExtractRootPackageFileTest(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "root_packages") + + @classmethod + def tearDownClass(cls): + del cls.fs + super().tearDownClass() + + def test_file_not_found(self): + with self.assertRaises(payload.FileNotFoundError) as err: + payload.extract_root_package_data(self.fs, "does_not_exist.toml") + assert err.path == "does_not_exist.toml" + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as err: + payload.extract_root_package_data(self.fs, "broken.toml") + assert err.path == "broken.toml" + + def test_fields_not_in_table(self): + with self.assertRaises(payload.FieldsNotInTable) as err: + payload.extract_root_package_data(self.fs, "fields_not_in_table.toml") + assert err.path == "fields_not_in_table.toml" + assert err.fields == ["created_by", "format_version"] + + def test_missing_meta_table(self): + with self.assertRaises(payload.TableNotFoundError) as err: + payload.extract_root_package_data(self.fs, "missing_meta.toml") + assert err.path == "missing_meta.toml" + assert err.table == "meta" + assert "[meta]" in str(err) + + def test_missing_learning_package_table(self): + with self.assertRaises(payload.TableNotFoundError) as err: + payload.extract_root_package_data(self.fs, "missing_learning_package.toml") + assert err.path == "missing_learning_package.toml" + assert err.table == "learning_package" + assert "[learning_package]" in str(err) + + def test_unsupported_format_version(self): + # We don't support format_version=2 + with self.assertRaises(payload.UnsupportedFormatError) as err: + payload.extract_root_package_data( + self.fs, "unsupported_format_version_2.toml" + ) + # We don't support format_version as anthing other than number + with self.assertRaises(payload.UnsupportedFormatError) as err: + payload.extract_root_package_data( + self.fs, "unsupported_format_version_b.toml" + ) + + # We will allow format_version 1.x though, in case we want to extend our + # format in a fully backwards compatible way. + root_data = payload.extract_root_package_data( + self.fs, "unsupported_format_version_1_1.toml" + ) + assert root_data["meta"]["format_version"] == 1.1 + + def test_ignore_unknown_tables(self): + """Allow for forwards compatibility.""" + assert "unknown" in payload.extract_root_package_data( + self.fs, "unknown_table.toml" + ) + + def test_minimal(self): + data = payload.extract_root_package_data(self.fs, "minimal.toml") + assert data == { + "meta": { + "format_version": 1, + }, + "learning_package": {}, + } + + def test_normal(self): + data = payload.extract_root_package_data(self.fs, "normal_ulmo_v1.toml") + assert data == { + "meta": { + "format_version": 1, + "created_by": "eddy", + "created_by_email": "eddy@axim.org", + "created_at": datetime( + 2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc + ), + "origin_server": "studio.local.openedx.io", + }, + "learning_package": { + "title": "Fun Library", + "key": "lib:Axim:FunLib", + "description": "My very fun library! 🐢", + "created": datetime( + 2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc + ), + "updated": datetime( + 2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc + ), + }, + } + + +class ExtractEntityDataTest(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "entities") + + @classmethod + def tearDownClass(cls): + del cls.fs + super().tearDownClass() + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as err: + payload.extract_entity_data(self.fs, "broken.toml") + + def test_missing_entity_table(self): + with self.assertRaises(payload.TableNotFoundError) as err: + payload.extract_entity_data(self.fs, "missing_entity_table.toml") + assert err.path == "missing_entity_table.toml" + assert err.table == "entity" + assert "[entity]" in str(err) + + def test_missing_entity_key(self): + with self.assertRaises(payload.FieldMissing) as err: + payload.extract_entity_data(self.fs, "missing_entity_key.toml") + assert err.missing_field == "key" + assert err.table == "entity" + + def test_dupes(self): + """ + Test for duplicate entities. + + If we didn't explicitly check for this, a second file defining the same + entity one entity would just overwrite + the other, which would confuse people who might be assembling an archive + file for restoring. + + This test is different from the others because extract_entities_data + doesn't raise exceptions, it collects them from its calls to + extract_entity_data(). + """ + paths = ["dupe_1.toml", "dupe_2.toml"] + data, _path_mapping, errors = payload.extract_entities_data(self.fs, paths) + assert "dupe-key" in data # The first one should have succeeded... + assert len(data) == 1 # but the duplicate never made it in. + assert len(errors) == 1 # There should be only one error. + + error = errors[0] + assert error.original_path == "dupe_1.toml" # path of the original + assert error.path == "dupe_2.toml" # path where error was marked + + @skip + def test_ignore_unknown_tables(self): + # assert "unknown" in payload.extract_root_package_data(self.fs, "unknown_table.toml") + pass + + @skip + def test_normal_component(self): + pass + + def test_normal_container(self): + ref, data = payload.extract_entity_data(self.fs, "normal_container.toml") + assert ref == "section-9-ac4b9f" + assert data == { + 'can_stand_alone': True, + 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), + 'draft': { + 'version_num': 2 + }, + 'published': { + 'version_num': 1 + }, + 'container': { + 'section': {} + }, + 'versions': [ + { + 'title': 'Section 9', + 'version_num': 2, + 'container': { + 'children': [ + 'week-7-e73782', + 'subsection-001-e4bbe5' + ] + } + }, + { + 'title': 'Section 9', + 'version_num': 1, + 'container': { + 'children': [ + 'week-7-e73782' + ] + } + } + ] + } + + diff --git a/tests/openedx_content/applets/backup_restore/test_restore.py b/tests/openedx_content/applets/backup_restore/test_restore.py index fcaf99acd..693e886cf 100644 --- a/tests/openedx_content/applets/backup_restore/test_restore.py +++ b/tests/openedx_content/applets/backup_restore/test_restore.py @@ -291,7 +291,7 @@ def test_error_learning_package_missing_key(self): # Mock parse_learning_package_toml to return a dict without 'key' with patch( - "openedx_content.applets.backup_restore.zipper.parse_learning_package_toml", + "openedx_content.applets.backup_restore.zipper.LearningPackageUnzipper.extract_root_package_data", return_value={ "learning_package": { "title": "Library test", @@ -322,7 +322,7 @@ def test_error_no_metadata_section(self): # Mock parse_learning_package_toml to return a dict without 'meta' with patch( - "openedx_content.applets.backup_restore.zipper.parse_learning_package_toml", + "openedx_content.applets.backup_restore.zipper.LearningPackageUnzipper.extract_root_package_data", return_value={ "learning_package": { "title": "Library test", @@ -383,6 +383,14 @@ def test_success_metadata_using_user_context(self): assert metadata == expected_metadata +from textwrap import dedent + +class RestoreV2TestCase(RestoreTestCase): + + def test_package_toml_parsing(self): + pass + + class RestoreUtilitiesTest(TestCase): """Tests for utility functions used in the restore process.""" From ebc88ba0d675983a46cb8410d76d742e802d2d85 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 10:56:20 -0400 Subject: [PATCH 02/14] temp: load in transaction, improve comments --- .../applets/backup_restore/api.py | 18 ++---- .../applets/backup_restore/loading.py | 64 ++++++++++--------- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 11f2217e6..786a798d9 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -1,13 +1,8 @@ """ Backup Restore API -Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage - -Extract -> Validate -> Load - - -(FS + root) -> UnvalidatedLearningPackage -> ValidatedLearningPackageInput - +This module is responsible for creating a backup archive of a Learning Package, +as well as creating a new Learning Package based on a backup archive file. """ from datetime import datetime, timezone @@ -38,17 +33,18 @@ def load_learning_package( Restores the learning package and its contents to the database. The overall pipeline looks like this: + Archive location (Path) → FileSystem (fsspec) → UnvalidatedLearningPackageInput → ValidatedLearningPackageInput → LearningPackage - TODO: Returns a dictionary with the status of the operation and any errors encountered. + Loads a learning package from a zip file at the given path. Restores the + learning package and its contents to the database. - Loads a learning package from a zip file at the given path. - Restores the learning package and its contents to the database. - Returns a dictionary with the status of the operation and any errors encountered. + Returns a dictionary with the status of the operation and any errors + encountered during that process. """ fs = archive.read_fs_for_path(path_str) unvalidated_input = payload.extract_unvalidated_learning_package(fs) diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py index 984743844..61e51c6be 100644 --- a/src/openedx_content/applets/backup_restore/loading.py +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -10,6 +10,7 @@ from functools import cache, partial from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user +from django.db.transaction import atomic from ..components import api as components_api from ..containers import api as containers_api @@ -74,39 +75,40 @@ def load_into(self, target: Target): incremental imports where we have to test the same input being imported into multiple Learning Packages with existing state. """ - bulk_change_context_for_time = partial( - publishing_api.bulk_draft_changes_for, - target.learning_package.id, - changed_by=target.user.id, - ) + with atomic(savepoint=False): + bulk_change_context_for_time = partial( + publishing_api.bulk_draft_changes_for, + target.learning_package.id, + changed_by=target.user.id, + ) - # DraftChangeLog 1: Add all the PublishableEntities and their versions, - # and set their versions to prepare for for publishing. - with bulk_change_context_for_time(changed_at=target.loaded_at): - loaded_components = self.load_components_into(target) - loaded_entities = self.load_containers_into(target, loaded_components) - self.set_draft_versions(target, for_publishing=True) - - publishing_api.publish_all_drafts( - target.learning_package.id, - published_at=target.loaded_at, - published_by=target.user.id, - message="Restore from backup.", - ) + # DraftChangeLog 1: Add all the PublishableEntities and their versions, + # and set their versions to prepare for for publishing. + with bulk_change_context_for_time(changed_at=target.loaded_at): + loaded_components = self.load_components_into(target) + loaded_entities = self.load_containers_into(target, loaded_components) + self.set_draft_versions(target, for_publishing=True) + + publishing_api.publish_all_drafts( + target.learning_package.id, + published_at=target.loaded_at, + published_by=target.user.id, + message="Restore from backup.", + ) - # DraftChangeLog 2: Set all PublishableEntities to their proper draft. - # At this point, all versions have been loaded, and the correct versions - # have been published, but the current draft version might be wrong. - # - # The history display will want draft changes to be slightly after the - # published log entry. - changed_at = target.loaded_at + timedelta(seconds=1) - with bulk_change_context_for_time(changed_at=changed_at): - self.set_draft_versions(target, for_publishing=False) - - # Collections are added at the end, in case publishing of contents would - # cause more thrashing w.r.t. search indexing. - self.load_collections_into(target, loaded_entities) + # DraftChangeLog 2: Set all PublishableEntities to their proper draft. + # At this point, all versions have been loaded, and the correct versions + # have been published, but the current draft version might be wrong. + # + # The history display will want draft changes to be slightly after the + # published log entry. + changed_at = target.loaded_at + timedelta(seconds=1) + with bulk_change_context_for_time(changed_at=changed_at): + self.set_draft_versions(target, for_publishing=False) + + # Collections are added at the end, in case publishing of contents would + # cause more thrashing w.r.t. search indexing. + self.load_collections_into(target, loaded_entities) return self.build_restore_result(target) From be50716b6505fcedc3acc188390acc0c4e4165b3 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 11:38:12 -0400 Subject: [PATCH 03/14] temp: updating comments --- .../applets/backup_restore/api.py | 43 +++++++++++-------- .../applets/backup_restore/archive.py | 6 +-- .../applets/backup_restore/payload.py | 17 +++++--- .../applets/backup_restore/readme.rst | 28 +----------- 4 files changed, 41 insertions(+), 53 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 786a798d9..aeeba3599 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -14,7 +14,9 @@ from . import archive, loading, payload, validation -from .zipper import LearningPackageZipper, generate_staged_package_ref +from .zipper import ( + LearningPackageZipper, RestoreResult, generate_staged_package_ref, zipfile, LearningPackageUnzipper +) @attrs.define(frozen=True) @@ -28,21 +30,23 @@ def load_learning_package( package_ref: str | None = None, ) -> dict: """ - Loads a learning package from a zip file at the given path. - - Restores the learning package and its contents to the database. - - The overall pipeline looks like this: + Loads a learning package from a file system at the given path. - Archive location (Path) → - FileSystem (fsspec) → - UnvalidatedLearningPackageInput → - ValidatedLearningPackageInput → - LearningPackage + The ``path_str`` will usually point to a Zip file archive that holds the + backup of the Learning Package data. For testing and debugging purposes, you + can also specify ``path_str`` to be the root directory of an unzipped + version of the archive data. Loads a learning package from a zip file at the given path. Restores the learning package and its contents to the database. + The overall pipeline looks like this: + + archive.py: Archive location (Path) → FileSystem (fsspec) + payload.py: FileSystem (fsspec) → UnvalidatedLearningPackageInput + validation.py: UnvalidatedLearningPackageInput → ValidatedLearningPackageInput + loading.py: ValidatedLearningPackageInput → LearningPackage + Returns a dictionary with the status of the operation and any errors encountered during that process. """ @@ -73,15 +77,16 @@ def load_learning_package( return result -def pretty_print(obj): - from pydantic import TypeAdapter - from typing import Any - from rich import print_json - - print_json(TypeAdapter(Any).dump_json(obj, indent=2).decode("utf8")) - - ### This was pre-existing: +def load_learning_package_old(path: str, package_ref: str | None = None, user: UserType | None = None) -> dict: + """ + Loads a learning package from a zip file at the given path. + Restores the learning package and its contents to the database. + Returns a dictionary with the status of the operation and any errors encountered. + """ + with zipfile.ZipFile(path, "r") as zipf: + return LearningPackageUnzipper(zipf, package_ref, user).load() + def create_zip_file( lp_key: str, diff --git a/src/openedx_content/applets/backup_restore/archive.py b/src/openedx_content/applets/backup_restore/archive.py index 418739a13..ee0149d10 100644 --- a/src/openedx_content/applets/backup_restore/archive.py +++ b/src/openedx_content/applets/backup_restore/archive.py @@ -20,13 +20,13 @@ def read_fs_for_path(path_str: str) -> AbstractFileSystem: implications, and I don't want to open the door on "supported archive formats" to include everything under the sun. But it's an intriguing option to consider. - - TODO: Can we force read-only mode on these file systems? """ path = Path(path_str) if path.is_dir(): + # read-only mode is not available for DirFileSystem return DirFileSystem(path) elif path.is_file() and path.suffix.lower() == ".zip": - return ZipFileSystem(path) + # read-only is the default for ZipFilesystem, but make it explicit + return ZipFileSystem(path, mode="r") raise ValueError(f"Could not load path {path_str}") diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index b6e5d1135..4599291f1 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -3,11 +3,8 @@ the archive container format that the files are bundled in, e.g. a local file system directory, a zip file archive, or something more exotic down the line. -Some high level considerations for this module: - -1. The error checking is for the file format itself, i.e. extracting values - from the TOML files and static assets and assembling them for validation. - In some cases, this means we do have to look for particular fields to handle +The error checking here is for the file format itself, i.e. extracting values +from the TOML files and static assets and assembling them for validation. """ from __future__ import annotations @@ -100,11 +97,21 @@ class UnsupportedFormatError(ExtractionError): class PayloadExtractor: """ Extracts files from a file system and generates unvalidated input. + + TODO: This is not in use yet, but we want to eventually place extraction- + related functionality in this class so that it's easier to swap out + extraction behavior with other classes later, e.g. if people have different + formatting ideas for their archive formats. """ def __init__(self, fs: AbstractFileSystem): self.fs = fs + # TODO: This is not complete. The problem we eventually want to solve is + # that sometimes people archive the right thing and package.toml is at + # the root, but sometimes people make an archive that has one folder in + # it, and everything (including the package.toml) is in that folder. So + # we want to gracefully accept that format and have our root change. if self.fs.exists("package.toml"): self.root = "" elif len(fs.ls('.')) == 1: diff --git a/src/openedx_content/applets/backup_restore/readme.rst b/src/openedx_content/applets/backup_restore/readme.rst index 687bfea68..d9dcccf5b 100644 --- a/src/openedx_content/applets/backup_restore/readme.rst +++ b/src/openedx_content/applets/backup_restore/readme.rst @@ -1,30 +1,6 @@ Backup/Restore Applet ===================== -The ``backup_restore`` is responsible for making a backup archive of an existing Learning Package, or creating a new Learning Package from an existing archive. +The ``backup_restore`` applet is responsible for making a backup archive of an existing Learning Package, or creating a new Learning Package from an existing archive. -Motivation ----------- - - -Intended Use Cases ------------------- - - - -Architecture Guidelines ------------------------ - - - -Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage - -Extract -> Validate -> Load - - -Archive - -We are very intentionally separating the following aspects: - -archive.py - The actual \ No newline at end of file +The codebase is currently in a transitional phase where we are moving towards pydantic schemas for more robust validation and error checking. From 85bdff061f03ebc93ea872444e61f374d2df08ac Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 13:10:29 -0400 Subject: [PATCH 04/14] feat!: complete the pydantic restore path for backup_restore The restore pipeline (archive -> payload -> validation -> loading) was in place but had never run end to end. This finishes it, gives it a single error model, and covers it with tests. loading.py predated the "keys are now opaque refs" rename (5d4fbf4) and its siblings, so it still passed `key=`/`local_key=` and read `.key` off models that now expose `package_ref`/`entity_ref`/`component_code`/`collection_code`. mypy reported 38 errors against this applet; it now reports none. Error handling -------------- All errors now descend from a single `BackupRestoreError` in the new errors.py. The existing `ExtractionError` classes move there and re-parent onto it, so there is no translation layer between error models. `validate()` no longer lets a raw pydantic ValidationError escape the public API, and no longer discards the extraction errors that payload.py had carefully collected. It gathers everything -- extraction errors as-is, one `SchemaError` per pydantic entry, plus consistency checks -- and reports it all at once, so someone repairing an archive by hand sees every problem in a single pass. Pydantic locations are mapped back to the archive file they came from, which is what `entity_path_mapping` was always for. The five consistency checks (unresolved child, missing draft/published version, duplicate version_num, malformed component ref, unknown container type) were each an uncaught exception in the middle of a database write. Bugs fixed along the way ------------------------ * The container union could not discriminate. All three container models default their single field and inherit `extra="allow"`, so every one of them validated every dict and the leftmost union member always won -- meaning every container would have loaded as a Unit, silently. * REF_CONSTRAINTS and CODE_CONSTRAINTS had trailing commas, making them 1-tuples, so strictness, whitespace stripping and the code regex were inert. * Restored packages were named "Temp Title"; title, description and created now come from the archive. * lp_dump writes `user.email` unconditionally and Django defaults it to "", so a backup taken by a user with no email produced an archive our own restore rejected. Blank [meta] strings now read as absent. * `[meta]` fields other than format_version were effectively required. * CollectionInput declared `created` twice and never declared `entities`, which loading.py read via `extra="allow"`. API changes ----------- `load_learning_package` now returns a `RestoreResult` and raises `RestoreFailedError`. `load_learning_package_as_dict` is a compatibility shim returning the old dict shape for callers that still expect it. api.py declares `__all__` again -- it had stopped, which leaked `zipfile`, `attrs`, `atomic` and the pipeline modules into `openedx_content.api`. lp_load now uses the new pipeline, accepts a directory as well as a .zip, and takes an optional --package-ref. lp_load2 and load_learning_package_old are gone, as is the scratch `encode` command (it called an unimported tomli_w). Retiring the old read path -------------------------- LearningPackageUnzipper, serializers.py and toml.py's parse functions are removed. The write side (LearningPackageZipper, toml.py's writers, create_zip_file) is untouched and still to be migrated. Tests ----- test_restore.py is replaced by test_loading.py, which ports every case it covered plus directory loading, the draft/published resolution matrix, static asset round-tripping, and rollback-on-failure. Two of the old tests had been failing since they patched a method that does not exist; those cases now use real fixtures instead of mocks. New test_archive.py, test_schema.py and test_validation.py; test_payload.py gains collection extraction, whole-archive assembly, and the seven fixture files that were committed empty. Assertions that sat unreachable inside `assertRaises` blocks now actually run. 33 passing / 2 failing / 2 skipped -> 139 passing. Co-Authored-By: Claude Opus 5 (1M context) --- mypy.ini | 3 + pytest.ini | 4 - .../applets/backup_restore/api.py | 94 ++- .../applets/backup_restore/archive.py | 9 +- .../applets/backup_restore/errors.py | 206 +++++ .../applets/backup_restore/loading.py | 124 ++- .../applets/backup_restore/payload.py | 129 ++- .../applets/backup_restore/readme.rst | 9 +- .../applets/backup_restore/results.py | 111 +++ .../applets/backup_restore/schema.py | 100 +-- .../applets/backup_restore/serializers.py | 216 ----- .../applets/backup_restore/toml.py | 25 - .../applets/backup_restore/validation.py | 221 ++++- .../applets/backup_restore/zipper.py | 762 +----------------- .../management/commands/encode.py | 98 --- .../management/commands/lp_load.py | 68 +- .../management/commands/lp_load2.py | 67 -- test_utils/zip_file_utils.py | 25 + .../duplicate_entities/entities/first.toml | 15 + .../duplicate_entities/entities/second.toml | 17 + .../broken/duplicate_entities/package.toml | 13 + .../fixtures/broken/empty_archive/.gitkeep | 1 + .../broken/missing_lp_key/package.toml | 12 + .../fixtures/broken/missing_meta/package.toml | 7 + .../unknown_container/entities/mystery.toml | 16 + .../broken/unknown_container/package.toml | 13 + .../unresolved_child/entities/unit1.toml | 19 + .../broken/unresolved_child/package.toml | 13 + .../unsupported_format_version/package.toml | 11 + .../broken_collection/collections/broken.toml | 3 + .../broken_collection/package.toml | 15 + .../payload_test_data/collections/broken.toml | 3 + .../payload_test_data/collections/dupe_1.toml | 6 + .../payload_test_data/collections/dupe_2.toml | 8 + .../collections/fields_not_in_table.toml | 8 + .../collections/missing_collection_table.toml | 7 + .../payload_test_data/collections/normal.toml | 9 + .../duplicate_entities/entities/dupe_1.toml | 4 + .../duplicate_entities/entities/dupe_2.toml | 4 + .../payload_test_data/empty_archive/.gitkeep | 2 + .../entities/missing_versions.toml | 13 + .../entities/normal_component.toml | 20 + .../component_versions/v2/block.xml | 1 + .../component_versions/v3/block.xml | 1 + .../component_versions/v3/static/figure.png | 1 + .../entities/unknown_table.toml | 26 + .../unsupported_format_version_true.toml | 8 + .../applets/backup_restore/test_archive.py | 69 ++ .../applets/backup_restore/test_loading.py | 579 +++++++++++++ .../applets/backup_restore/test_payload.py | 268 +++++- .../applets/backup_restore/test_restore.py | 414 ---------- .../applets/backup_restore/test_schema.py | 340 ++++++++ .../applets/backup_restore/test_validation.py | 355 ++++++++ tox.ini | 4 + 54 files changed, 2704 insertions(+), 1872 deletions(-) delete mode 100644 pytest.ini create mode 100644 src/openedx_content/applets/backup_restore/errors.py create mode 100644 src/openedx_content/applets/backup_restore/results.py delete mode 100644 src/openedx_content/applets/backup_restore/serializers.py delete mode 100644 src/openedx_content/management/commands/encode.py delete mode 100644 src/openedx_content/management/commands/lp_load2.py create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml create mode 100644 tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml create mode 100644 tests/openedx_content/applets/backup_restore/test_archive.py create mode 100644 tests/openedx_content/applets/backup_restore/test_loading.py delete mode 100644 tests/openedx_content/applets/backup_restore/test_restore.py create mode 100644 tests/openedx_content/applets/backup_restore/test_schema.py create mode 100644 tests/openedx_content/applets/backup_restore/test_validation.py diff --git a/mypy.ini b/mypy.ini index b383a8816..f3994e68d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,5 +12,8 @@ files = [mypy-organizations.*] follow_untyped_imports = True +[mypy-fsspec.*] +follow_untyped_imports = True + [mypy.plugins.django-stubs] django_settings_module = "projects.dev" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 3621a80a5..000000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -testpaths = ["tests"] -pythonpath = src -DJANGO_SETTINGS_MODULE = test_settings diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index aeeba3599..e2d4b32d2 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -4,31 +4,31 @@ This module is responsible for creating a backup archive of a Learning Package, as well as creating a new Learning Package based on a backup archive file. """ +from dataclasses import asdict from datetime import datetime, timezone +from io import StringIO -import attrs from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user from django.db.transaction import atomic from ..publishing import api as publishing_api from . import archive, loading, payload, validation +from .errors import BackupRestoreError, RestoreFailedError +from .results import RestoreResult, generate_staged_package_ref +from .zipper import LearningPackageZipper - -from .zipper import ( - LearningPackageZipper, RestoreResult, generate_staged_package_ref, zipfile, LearningPackageUnzipper -) - - -@attrs.define(frozen=True) -class ImportResult: - entities_created: int # Should this be a list of entity refs instead? +__all__ = [ + "create_zip_file", + "load_learning_package", + "load_learning_package_as_dict", +] def load_learning_package( path_str: str, user: UserType, package_ref: str | None = None, -) -> dict: +) -> RestoreResult: """ Loads a learning package from a file system at the given path. @@ -37,9 +37,6 @@ def load_learning_package( can also specify ``path_str`` to be the root directory of an unzipped version of the archive data. - Loads a learning package from a zip file at the given path. Restores the - learning package and its contents to the database. - The overall pipeline looks like this: archive.py: Archive location (Path) → FileSystem (fsspec) @@ -47,29 +44,40 @@ def load_learning_package( validation.py: UnvalidatedLearningPackageInput → ValidatedLearningPackageInput loading.py: ValidatedLearningPackageInput → LearningPackage - Returns a dictionary with the status of the operation and any errors - encountered during that process. + If ``package_ref`` is not supplied, we generate a staged one namespaced to + ``user``. We can't just use the ref from the archive, because the archive can + claim any ref it likes and the user may not be allowed to create it. + + Errors that can be raised: + + * ``ArchiveNotReadableError`` if we can't open ``path_str`` at all. + * ``RestoreFailedError`` if the archive's contents don't validate. Nothing is + written to the database in that case. + + Both descend from ``BackupRestoreError``. """ fs = archive.read_fs_for_path(path_str) unvalidated_input = payload.extract_unvalidated_learning_package(fs) - - # TODO: need to be able to exit early here if errors make the rest of this - # pointless. The Loader class currently knows how to make output that we can - # send up to platform, but maybe that knowledge should be in this module - # instead? - # if unvalidated_input.errors: validated_input = validation.validate(unvalidated_input) - if package_ref is None: - package_ref = generate_staged_package_ref( - validated_input.data.learning_package.key, user, - ) + # Bail out before touching the database. We deliberately don't do a partial + # restore: a half-loaded Learning Package is harder to reason about than no + # Learning Package at all. + if validated_input.errors: + raise RestoreFailedError(validated_input.errors) loader = loading.Loader(validated_input) + archive_lp = loader.data.learning_package + if package_ref is None: + package_ref = generate_staged_package_ref(archive_lp.key, user) + now = datetime.now(tz=timezone.utc) with atomic(savepoint=False): learning_package = publishing_api.create_learning_package( - package_ref, "Temp Title", created=now + package_ref, + archive_lp.title, + description=archive_lp.description or "", + created=archive_lp.created or now, ) load_target = loading.Loader.Target(learning_package, user, now) result = loader.load_into(load_target) @@ -77,15 +85,33 @@ def load_learning_package( return result -### This was pre-existing: -def load_learning_package_old(path: str, package_ref: str | None = None, user: UserType | None = None) -> dict: +def load_learning_package_as_dict( + path_str: str, + user: UserType, + package_ref: str | None = None, +) -> dict: """ - Loads a learning package from a zip file at the given path. - Restores the learning package and its contents to the database. - Returns a dictionary with the status of the operation and any errors encountered. + ``load_learning_package``, in the dict shape the frontend currently expects. + + Returns a dict with the status of the operation and any errors encountered + during that process, rather than raising. + + TODO: This exists so that callers written against the pre-pydantic restore + keep working. New callers should use ``load_learning_package`` and catch + ``BackupRestoreError``, so that this can eventually go away. """ - with zipfile.ZipFile(path, "r") as zipf: - return LearningPackageUnzipper(zipf, package_ref, user).load() + try: + result = load_learning_package(path_str, user, package_ref) + except RestoreFailedError as err: + return asdict( + RestoreResult(status="error", log_file_error=StringIO(err.as_text())) + ) + except BackupRestoreError as err: + return asdict( + RestoreResult(status="error", log_file_error=StringIO(f"{err}\n")) + ) + + return asdict(result) def create_zip_file( diff --git a/src/openedx_content/applets/backup_restore/archive.py b/src/openedx_content/applets/backup_restore/archive.py index ee0149d10..1538965b9 100644 --- a/src/openedx_content/applets/backup_restore/archive.py +++ b/src/openedx_content/applets/backup_restore/archive.py @@ -4,9 +4,12 @@ """ from pathlib import Path +from fsspec import AbstractFileSystem from fsspec.implementations.dirfs import DirFileSystem from fsspec.implementations.zip import ZipFileSystem -from fsspec import AbstractFileSystem + +from .errors import ArchiveNotReadableError + def read_fs_for_path(path_str: str) -> AbstractFileSystem: """ @@ -29,4 +32,6 @@ def read_fs_for_path(path_str: str) -> AbstractFileSystem: # read-only is the default for ZipFilesystem, but make it explicit return ZipFileSystem(path, mode="r") - raise ValueError(f"Could not load path {path_str}") + raise ArchiveNotReadableError( + "Expected a directory or a .zip file", path=path_str + ) diff --git a/src/openedx_content/applets/backup_restore/errors.py b/src/openedx_content/applets/backup_restore/errors.py new file mode 100644 index 000000000..9f34d8bb1 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/errors.py @@ -0,0 +1,206 @@ +""" +Errors raised while backing up or restoring a Learning Package. + +Everything in this applet raises something that descends from +:class:`BackupRestoreError`, so callers that don't care about the specifics can +catch a single type. The subclasses exist to give useful debug output and to let +tests assert on a specific failure rather than on message text. + +The hierarchy is grouped by the pipeline stage that detects the problem: + +* :class:`ArchiveNotReadableError` -- ``archive.py``, we can't open the thing at all. +* :class:`ExtractionError` -- ``payload.py``, the archive's files are malformed in a + way that stops us assembling the input document (unparseable TOML, duplicate + entity definitions, missing mandatory files). +* :class:`SchemaError` -- ``validation.py``, a field failed pydantic validation. +* :class:`ConsistencyError` -- ``validation.py``, a cross-reference check that + pydantic can't express (e.g. a container pointing at a child that isn't in the + archive). + +:class:`RestoreFailedError` is the aggregate that the public API raises. It holds +all of the individual errors found during a single restore attempt. +""" +from __future__ import annotations + + +class BackupRestoreError(Exception): + """ + Base class for every error this applet raises. + + Args: + message: Human-readable description of what went wrong. + path: Archive-relative path of the file the problem was found in, e.g. + ``"entities/unit1-b7eafb.toml"``. ``None`` when the error isn't + attributable to a particular file. + """ + + def __init__(self, message, path=None): + super().__init__(message) + self.message = message + self.path = path + + def __str__(self): + return f"{self.path}: {self.message}" + + +class ArchiveNotReadableError(BackupRestoreError): + """We could not open the archive at all (not a directory, not a zip file).""" + + +# --- Extraction Errors (payload.py) --- + + +class ExtractionError(BackupRestoreError): + """ + Any error during the extraction process. + + At the moment, any error is fatal. The point of the different errors is to + provide useful debug logging and to let us write tests that look for + specific errors. + """ + + +class InvalidTOMLError(ExtractionError): + def __init__(self, file_description, details, path): + message = f"Cannot decode TOML for {file_description}: {details}" + super().__init__(message, path=path) + + +class TableNotFoundError(ExtractionError): + def __init__(self, file_description, table, path): + self.table = table + message = f"Table [{table}] not found in {file_description}." + super().__init__(message, path=path) + + +class FieldsNotInTable(ExtractionError): + def __init__(self, file_description, fields, path): + self.fields = sorted(fields) + message = f"{file_description} has fields not in a table: {', '.join(fields)}" + super().__init__(message, path=path) + + +class FieldMissing(ExtractionError): + """A table is missing a field we need in order to go on.""" + + def __init__(self, file_description, table, missing_field, path): + self.table = table + self.missing_field = missing_field + message = ( + f'{file_description} is missing required field "{missing_field}" ' + f"from table [{table}]" + ) + super().__init__(message, path=path) + + +class MissingFileError(ExtractionError): + """ + A file we require is not in the archive. + + Note: this used to be called ``FileNotFoundError``, which shadowed the + builtin of the same name and made ``except FileNotFoundError`` ambiguous for + our callers. + """ + + def __init__(self, file_description, path): + message = f"{file_description} file not found at expected path" + super().__init__(message, path=path) + + +class DuplicateFoundError(ExtractionError): + def __init__(self, description, original_path, path): + self.original_path = original_path + message = f"{description} already defined in {original_path}" + super().__init__(message, path=path) + + +class UnsupportedFormatError(ExtractionError): + """The archive declares a ``format_version`` we don't know how to read.""" + + +# --- Validation Errors (validation.py) --- + + +class SchemaError(BackupRestoreError): + """ + A field failed pydantic validation. + + One of these is created for each entry in a ``pydantic.ValidationError``, so + that we can attribute the failure to the archive file it came from instead + of reporting a JSON pointer into a document the user never sees. + + Args: + message: pydantic's error message for this entry. + path: Archive-relative path of the source file, if we can work it out. + location: The part of pydantic's ``loc`` tuple that is meaningful + *within* that file, e.g. ``("versions", 0, "title")``. + """ + + def __init__(self, message, path=None, location=()): + self.location = tuple(location) + super().__init__(message, path=path) + + def __str__(self): + if self.location: + location_str = ".".join(str(part) for part in self.location) + return f"{self.path}: {location_str}: {self.message}" + return super().__str__() + + +class ConsistencyError(BackupRestoreError): + """ + A cross-reference in the archive doesn't hold up. + + These are the checks that pydantic can't express, because they involve more + than one part of the document at once. + """ + + +class UnresolvedChildError(ConsistencyError): + """A container version lists a child that isn't defined in the archive.""" + + +class MissingVersionError(ConsistencyError): + """An entity's draft or published pointer names a version that isn't in the archive.""" + + +class DuplicateVersionError(ConsistencyError): + """An entity declares the same ``version_num`` more than once.""" + + +class MalformedRefError(ConsistencyError): + """An entity ref isn't in a shape we know how to load.""" + + +class UnknownContainerTypeError(ConsistencyError): + """The archive declares a container type this version of the code can't load.""" + + +# --- Aggregate --- + + +class RestoreFailedError(BackupRestoreError): + """ + The restore could not be completed. Holds every error we found. + + We deliberately gather as many problems as we can before raising, so that + someone fixing up an archive by hand doesn't have to discover their mistakes + one run at a time. + """ + + def __init__(self, errors: list[BackupRestoreError]): + self.errors = list(errors) + super().__init__(f"Restore failed with {len(self.errors)} error(s).") + + def __str__(self): + return self.as_text() + + def as_text(self) -> str: + """ + Render every error as a block of text suitable for a log file. + + The format matches what the pre-pydantic implementation wrote out, so + that existing consumers of the restore log keep working. + """ + lines = [str(err) for err in self.errors] + return "Errors encountered during restore:\n" + "\n".join(lines) + "\n" diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py index 61e51c6be..0382fb939 100644 --- a/src/openedx_content/applets/backup_restore/loading.py +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -4,31 +4,27 @@ """ import mimetypes import os.path - -from dataclasses import asdict, dataclass +from dataclasses import dataclass from datetime import datetime, timedelta from functools import cache, partial from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user from django.db.transaction import atomic +from ..collections import api as collections_api from ..components import api as components_api from ..containers import api as containers_api -from ..collections import api as collections_api from ..media import api as media_api +from ..media.models import Media from ..publishing import api as publishing_api from ..publishing.models import LearningPackage from ..sections.models import Section from ..subsections.models import Subsection from ..units.models import Unit - -from .schema import ( - SectionInputData, - SubsectionInputData, - UnitInputData, -) +from .errors import UnknownContainerTypeError +from .results import BackupMetadata, RestoreLearningPackageData, RestoreResult, unpack_package_ref +from .schema import EntityInputData, SectionInputData, SubsectionInputData, UnitInputData from .validation import ValidatedLearningPackageInput -from .zipper import RestoreResult, RestoreLearningPackageData, BackupMetadata class Loader: @@ -46,13 +42,18 @@ class Target: loaded_at: datetime def __init__(self, validated_input: ValidatedLearningPackageInput): + if validated_input.data is None: + raise ValueError( + "Cannot load a LearningPackage from input that failed validation." + ) self.validated_input = validated_input - self.component_inputs = {} - self.section_inputs = {} - self.subsection_inputs = {} - self.unit_inputs = {} + self.data = validated_input.data + self.component_inputs: dict[str, EntityInputData] = {} + self.section_inputs: dict[str, EntityInputData] = {} + self.subsection_inputs: dict[str, EntityInputData] = {} + self.unit_inputs: dict[str, EntityInputData] = {} - entities = validated_input.data.entities + entities = self.data.entities # Split our entities into separate dicts for convenience. for entity_ref, entity_input in sorted(entities.items()): @@ -66,8 +67,13 @@ def __init__(self, validated_input: ValidatedLearningPackageInput): case None: # For the moment, if it's not a Container, it's a Component self.component_inputs[entity_ref] = entity_input + case _: + # validation.py should have rejected this already. + raise UnknownContainerTypeError( + f'Cannot load entity "{entity_ref}": unrecognized container type', + ) - def load_into(self, target: Target): + def load_into(self, target: Target) -> RestoreResult: """ This method intentionally takes a target (LearningPackage, User, Datetime) instead of putting that information into Loader object state. @@ -112,17 +118,17 @@ def load_into(self, target: Target): return self.build_restore_result(target) - def build_restore_result(self, target: Target): + def build_restore_result(self, target: Target) -> RestoreResult: """ - This is for compatibility with what we're already sending to the frontend. + Summarize what we just loaded. - TODO: We should return something more structured for our API and let the - calling api.py handle the translation into what the REST API expects. + TODO: The shape of RestoreResult is inherited from what we already send + to the frontend. We should return something more structured here and let + api.py handle the translation into what the REST API expects. """ - validated_data = self.validated_input.data + validated_data = self.data - # Fix this with better parsing later. - _lib, org, slug = validated_data.learning_package.key.split(":") + org_code, package_code = unpack_package_ref(validated_data.learning_package.key) loaded_entities = publishing_api.get_publishable_entities(target.learning_package.id) @@ -131,10 +137,10 @@ def build_restore_result(self, target: Target): log_file_error=None, lp_restored_data=RestoreLearningPackageData( id=target.learning_package.id, - key=target.learning_package.key, - archive_lp_key=validated_data.learning_package.key, - archive_org_key=org, - archive_slug=slug, + package_ref=target.learning_package.package_ref, + archive_package_ref=validated_data.learning_package.key, + archive_org_code=org_code, + archive_package_code=package_code, title=target.learning_package.title, num_containers=loaded_entities.filter(container__isnull=False).count(), num_sections=loaded_entities.filter(container__section__isnull=False).count(), @@ -146,15 +152,20 @@ def build_restore_result(self, target: Target): backup_metadata=BackupMetadata( format_version=validated_data.meta.format_version, created_by=validated_data.meta.created_by, - created_by_email=validated_data.meta.created_by, + created_by_email=validated_data.meta.created_by_email, created_at=validated_data.meta.created_at, original_server=validated_data.meta.origin_server, ), ) - return asdict(result) + return result def load_components_into(self, target: Target): - """ """ + """ + Create every Component and its versions, and return them by ref. + + The returned mapping is what containers use to resolve their + children, so this has to run before any container is loaded. + """ @cache # inner fn, so won't persist across calls to load_components_into def _get_component_type(namespace: str, name: str): @@ -171,7 +182,7 @@ def _get_media_type(mime_type: str): component = components_api.create_component( target.learning_package.id, component_type=component_type, - local_key=component_code, + component_code=component_code, created=target.loaded_at, created_by=target.user.id, ) @@ -180,8 +191,11 @@ def _get_media_type(mime_type: str): entity_input.versions, key=lambda v: v.version_num ) for version_input in sorted_version_inputs: - media_to_replace = {} - for path, text_val in version_input.component.media.items(): + media_to_replace: dict[str, Media.ID | Media | bytes | None] = {} + version_media = ( + version_input.component.media if version_input.component else {} + ) + for path, text_val in version_media.items(): filename = os.path.basename(path) if filename == "block.xml": media_type = _get_media_type( @@ -196,7 +210,7 @@ def _get_media_type(mime_type: str): if path.startswith('static/'): # This is where we could add base64 encoded versions # right now, we just use fs:/path/to/file - _resource_type, filepath = text_val.split(":") + _resource_type, filepath = text_val.split(":", 1) new_media = media_api.get_or_create_file_media( target.learning_package.id, media_type.id, @@ -216,7 +230,7 @@ def _get_media_type(mime_type: str): # TODO: Modify create_next_component_version to take a Component # as an option, to save the needless fetches. components_api.create_next_component_version( - component.pk, + component.id, title=version_input.title, media_to_replace=media_to_replace, created=target.loaded_at, @@ -228,6 +242,12 @@ def _get_media_type(mime_type: str): return mapping def load_containers_into(self, target: Target, component_mapping: dict): + """ + Create every Container and its versions, and return them by ref. + + Containers are built bottom-up (Units, then Subsections, then + Sections), because each level references the one below it. + """ # Ordering matters, since we want to build the references bottom-up. container_types_to_inputs = { @@ -256,7 +276,11 @@ def load_containers_into(self, target: Target, component_mapping: dict): title=version_input.title, entities=[ mapping[child_ref] - for child_ref in version_input.container.children + for child_ref in ( + version_input.container.children + if version_input.container + else [] + ) ], created=target.loaded_at, created_by=target.user.id, @@ -268,36 +292,50 @@ def load_containers_into(self, target: Target, component_mapping: dict): return mapping def load_collections_into(self, target: Target, loaded_entities): - for collection_input in self.validated_input.data.collections: + """ + Create Collections and add their members. + + Members that aren't in the archive are skipped rather than treated + as an error -- a Collection with a missing entry is still usable. + """ + for collection_input in self.data.collections: collections_api.create_collection( target.learning_package.id, - key=collection_input.key, + collection_code=collection_input.key, title=collection_input.title, created_by=target.user.id, - description=collection_input.description, + description=collection_input.description or "", ) loaded_entity_refs = [ ref for ref in collection_input.entities if ref in loaded_entities ] entities = publishing_api.get_publishable_entities( target.learning_package.id - ).filter(key__in=loaded_entity_refs) + ).filter(entity_ref__in=loaded_entity_refs) collections_api.add_to_collection( target.learning_package.id, - key=collection_input.key, + collection_code=collection_input.key, entities_qset=entities, ) def set_draft_versions(self, target: Target, for_publishing: bool): - entity_inputs = self.validated_input.data.entities + """ + Point every entity's draft at the version the archive asks for. + + This runs twice. The first pass (``for_publishing=True``) sets each + draft to the version that should end up published, so that + ``publish_all_drafts`` publishes the right thing. The second pass + sets the drafts to their real values. + """ + entity_inputs = self.data.entities saved_entities = publishing_api.get_publishable_entities( target.learning_package.id ) for saved_entity in saved_entities: saved_draft_version = publishing_api.get_draft_version(saved_entity) - input_entity = entity_inputs[saved_entity.key] + input_entity = entity_inputs[saved_entity.entity_ref] if for_publishing: input_version_num = input_entity.published.version_num diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index 4599291f1..b2e7c9570 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -8,18 +8,36 @@ """ from __future__ import annotations -from numbers import Number + import os.path # fsspec doesn't work well with Path objects. import tomllib import attrs from fsspec import AbstractFileSystem +from .errors import ( + DuplicateFoundError, + ExtractionError, + FieldMissing, + FieldsNotInTable, + InvalidTOMLError, + MissingFileError, + TableNotFoundError, + UnsupportedFormatError, +) + ROOT_PACKAGE_PATH = "package.toml" @attrs.define(frozen=True) class UnvalidatedLearningPackageInput: + """ + Everything we could pull out of an archive, before validation. + + ``raw_data`` is the assembled document we hand to pydantic. + ``errors`` holds anything that stopped us assembling part of it. + """ + raw_data: dict errors: list[ExtractionError] fs: AbstractFileSystem @@ -28,72 +46,6 @@ class UnvalidatedLearningPackageInput: entity_path_mapping: dict[str, str] -class ExtractionError(Exception): - """ - Any error during the extraction process. - - At the moment, any error is fatal. The point of the different errors is to - provide useful debug logging and to let us write tests that look for - specific errors. - """ - - def __init__(self, message, path=None): - super().__init__(message) - self.message = message - self.path = path - - def __str__(self): - return f"{self.path}: {self.message}" - - -class InvalidTOMLError(ExtractionError): - def __init__(self, file_description, details, path): - message = f"Cannot decode TOML for {file_description}: {details}" - super().__init__(message, path=path) - - -class TableNotFoundError(ExtractionError): - def __init__(self, file_description, table, path): - self.table = table - message = f"Table [{table}] not found in {file_description}." - super().__init__(message, path=path) - - -class FieldsNotInTable(ExtractionError): - def __init__(self, file_description, fields, path): - self.fields = sorted(fields) - message = f"{file_description} has fields not in a table: {', '.join(fields)}" - super().__init__(message, path=path) - - -class FieldMissing(ExtractionError): - def __init__(self, file_description, table, missing_field, path): - self.table = table - self.missing_field = missing_field - message = ( - f'{file_description} is missing required field "{missing_field}" ' - f"from table [{table}]" - ) - super().__init__(message, path=path) - - -class FileNotFoundError(ExtractionError): - def __init__(self, file_description, path): - message = f"{file_description} file not found at expected path" - super().__init__(message, path=path) - - -class DuplicateFoundError(ExtractionError): - def __init__(self, description, original_path, path): - self.original_path = original_path - message = f"{description} already defined in {original_path}" - super().__init__(message, path=path) - - -class UnsupportedFormatError(ExtractionError): - pass - - class PayloadExtractor: """ Extracts files from a file system and generates unvalidated input. @@ -118,7 +70,6 @@ def __init__(self, fs: AbstractFileSystem): pass - def extract_unvalidated_learning_package( fs: AbstractFileSystem, ) -> UnvalidatedLearningPackageInput: @@ -154,8 +105,8 @@ def extract_unvalidated_learning_package( """ # The general philosophy here is to always march on and get as much as # possible, even if we know the upload is doomed. - unvalidated = {} - errors = [] + unvalidated: dict = {} + errors: list[ExtractionError] = [] # Root Package Metadata try: @@ -239,7 +190,7 @@ def extract_root_package_data(fs: AbstractFileSystem, path: str) -> dict: # Check: Root Package file exists at all. if not fs.exists(path): - raise FileNotFoundError(file_description, path=path) + raise MissingFileError(file_description, path=path) # Check: Is it a valid TOML file? with fs.open(path, "rb") as package_toml_file: @@ -264,7 +215,10 @@ def extract_root_package_data(fs: AbstractFileSystem, path: str) -> dict: # that is backwards compatible, i.e. it will reject 2 and higher, but accept # 1.1, 1.2, etc. format_version = root_package_dict["meta"].get("format_version") - if not isinstance(format_version, Number) or format_version >= 2: + is_number = isinstance(format_version, (int, float)) and not isinstance( + format_version, bool + ) + if not is_number or format_version >= 2: raise UnsupportedFormatError( f"Format version {format_version} is unsupported (only 1 is supported).", path=path, @@ -298,9 +252,16 @@ def get_entity_file_paths(fs: AbstractFileSystem) -> list[str]: def extract_entities_data(fs: AbstractFileSystem, paths: list[str]): - entities_data = {} - entity_path_mapping = {} - errors = [] + """ + Extract every entity file, collecting errors instead of raising them. + + Returns a ``(entities_data, entity_path_mapping, errors)`` tuple. The + path mapping lets later stages report errors against the file an entity + came from, which is not derivable from the entity ref. + """ + entities_data: dict[str, dict] = {} + entity_path_mapping: dict[str, str] = {} + errors: list[ExtractionError] = [] for entity_file_path in paths: try: entity_ref, entity_data = extract_entity_data( @@ -469,13 +430,21 @@ def extract_entity_data( def extract_collection_data(fs: AbstractFileSystem, path: str) -> dict: + """ + Extract the contents of a single Collection TOML file. + + We record the source path on the way out, so that a later duplicate-key + error can name both of the files involved. + """ file_description = "Collection" with fs.open(path, "rb") as collection_toml_file: try: collection_root_dict = tomllib.load(collection_toml_file) except tomllib.TOMLDecodeError as dec_err: - raise InvalidTOMLError(file_description, details=str(dec_err), path=path) + raise InvalidTOMLError( + file_description, details=str(dec_err), path=path + ) from dec_err _check_all_fields_in_tables(collection_root_dict, file_description, path) if "collection" not in collection_root_dict: @@ -508,11 +477,3 @@ def _check_all_fields_in_tables(data: dict, file_description, path): raise FieldsNotInTable( file_description, fields=fields_outside_of_tables, path=path ) - - -def pretty_print(obj): - from pydantic import TypeAdapter - from typing import Any - from rich import print_json - - print_json(TypeAdapter(Any).dump_json(obj, indent=2).decode("utf8")) diff --git a/src/openedx_content/applets/backup_restore/readme.rst b/src/openedx_content/applets/backup_restore/readme.rst index d9dcccf5b..e9b33abfc 100644 --- a/src/openedx_content/applets/backup_restore/readme.rst +++ b/src/openedx_content/applets/backup_restore/readme.rst @@ -3,4 +3,11 @@ Backup/Restore Applet The ``backup_restore`` applet is responsible for making a backup archive of an existing Learning Package, or creating a new Learning Package from an existing archive. -The codebase is currently in a transitional phase where we are moving towards pydantic schemas for more robust validation and error checking. +Restoring (reading an archive) is done with pydantic schema validation, in a pipeline of small modules that hand plain data to each other:: + + archive.py Archive location (a path) → FileSystem (fsspec) + payload.py FileSystem (fsspec) → UnvalidatedLearningPackageInput + validation.py UnvalidatedLearningPackageInput → ValidatedLearningPackageInput + loading.py ValidatedLearningPackageInput → LearningPackage + +Backing up (writing an archive) has not been migrated yet. It still lives in ``zipper.py`` and ``toml.py``, which build TOML directly from the Django models. The ``*OutputData`` models in ``schema.py`` are the beginning of that work, but nothing uses them yet. diff --git a/src/openedx_content/applets/backup_restore/results.py b/src/openedx_content/applets/backup_restore/results.py new file mode 100644 index 000000000..04f872661 --- /dev/null +++ b/src/openedx_content/applets/backup_restore/results.py @@ -0,0 +1,111 @@ +""" +The values we hand back after restoring a Learning Package from an archive. + +These live in their own module (rather than next to either the reading or the +writing code) because both the archive format and the database models change +independently of the summary we report to callers. + +TODO: ``RestoreResult`` and friends are shaped by what the frontend currently +expects, not by what's natural here. When we revisit the REST API, the loader +should return something structured and let ``api.py`` do the translation. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass +from datetime import datetime +from io import StringIO +from typing import Literal + +from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user + + +@dataclass +class RestoreLearningPackageData: + """ + Data about the restored learning package. + """ + id: int # The ID of the restored learning package + package_ref: str # The package_ref of the restored learning package (may be different if staged) + archive_package_ref: str # The original package_ref from the archive + archive_org_code: str | None # The org code parsed from archive_package_ref, or None if unparseable + archive_package_code: str | None # The package code parsed from archive_package_ref, or None if unparseable + title: str + num_containers: int + num_sections: int + num_subsections: int + num_units: int + num_components: int + num_collections: int + + +@dataclass +class BackupMetadata: + """ + Metadata about the backup operation. + """ + format_version: int + created_at: datetime | str | None + created_by: str | None = None + created_by_email: str | None = None + original_server: str | None = None + + +@dataclass +class RestoreResult: + """ + Result of the restore operation. + """ + status: Literal["success", "error"] + log_file_error: StringIO | None = None + lp_restored_data: RestoreLearningPackageData | None = None + backup_metadata: BackupMetadata | None = None + + +def unpack_package_ref(package_ref: str) -> tuple[str | None, str | None]: + """ + Try to parse org_code and package_code from a package_ref. + + By convention, package_refs take the form ``"{prefix}:{org_code}:{package_code}"``, + but this is only a convention — package_ref is opaque and the parse may fail. + Returns ``(None, None)`` if the ref does not match the expected format. + """ + parts = package_ref.split(":") + if len(parts) < 3: + return None, None + _, org_code, package_code = parts[:3] + return org_code, package_code + + +def generate_staged_package_ref(archive_package_ref: str, user: UserType) -> str: + """ + Generate a staged learning package ref based on the archive's package_ref. + + We can't trust package_ref from the archive directly, because the archive + could specify *any* arbitrary package_ref, and the user may or may not be + permitted to create an Package using that ref. So, instead, this function + generates a unique and semi-human-readable package_ref which is namespaced + to the current user and appropriate to provisionally save the package under. + The package_ref from the archive can then be presented to the user as a + *suggestion*, which they may or may not choose to use. + + Please note that the ref returned by this function is valid for Packages is + a generic sense, but it's not a valid Content Library key. Callers who are + restoring a Package for Library usage will need to replace this staged + package_ref before being able to render the Library's content. + + Arguments: + archive_package_ref (str): The original package_ref from the archive. + user (UserType | None): The user performing the restore operation. + + Example: + Input: "lib:WGU:LIB_C001" + Output: "lp-restore:dave:WGU:LIB_C001:1728575321" + """ + username = user.username + org_code, package_code = unpack_package_ref(archive_package_ref) + timestamp = int(time.time() * 1000) # Current time in milliseconds + if org_code and package_code: + return f"lp-restore:{username}:{org_code}:{package_code}:{timestamp}" + # Fallback for non-conventional package_refs + return f"lp-restore:{username}:{archive_package_ref}:{timestamp}" diff --git a/src/openedx_content/applets/backup_restore/schema.py b/src/openedx_content/applets/backup_restore/schema.py index 6bde51697..030e710a1 100644 --- a/src/openedx_content/applets/backup_restore/schema.py +++ b/src/openedx_content/applets/backup_restore/schema.py @@ -8,26 +8,27 @@ exports, and will be stricter. """ from __future__ import annotations + from pathlib import Path +from typing import Annotated, Literal from pydantic import ( AwareDatetime, BaseModel, ConfigDict, - Field, EmailStr, + Field, StrictStr, StringConstraints, - field_validator + field_validator, ) -from typing import Annotated, Literal # Refs are arbitrary identifiers that we do almost no validation of, and are # mainly there to assure uniqueness within some namespace. REF_CONSTRAINTS = StringConstraints( strict=True, strip_whitespace=True, -), +) # This is for things like the collection_code, library_code, etc. CODE_CONSTRAINTS = StringConstraints( @@ -39,7 +40,7 @@ # and strip_whitespace=True means that we're sure that we won't allow any # trailing newlines. pattern=r"^[a-zA-Z0-9_.-]+$", -), +) class InputData(BaseModel): @@ -83,7 +84,7 @@ def check_for_duplicate_keys(cls, collections: list[CollectionInput]): entries (and other broken entries), while still otherwise allowing the restore to proceed. But for now, any error kills the restore process. """ - collection_keys_to_paths = {} + collection_keys_to_paths: dict[str, CollectionInput] = {} for collection in collections: if collection.key in collection_keys_to_paths: originally_defined_collection = collection_keys_to_paths[collection.key] @@ -92,8 +93,7 @@ def check_for_duplicate_keys(cls, collections: list[CollectionInput]): f'{collection.src_path} (original in ' f'{originally_defined_collection.src_path})' ) - else: - collection_keys_to_paths[collection.key] = collection + collection_keys_to_paths[collection.key] = collection return collections @@ -115,10 +115,27 @@ class MetaInputData(InputData): affect input validation rules. """ format_version: Literal[1] # Only supported version at the moment - created_by: StrictStr | None = Field(min_length=1) - created_by_email: EmailStr | None - created_at: AwareDatetime | None - origin_server: StrictStr | None + created_by: StrictStr | None = Field(default=None, min_length=1) + created_by_email: EmailStr | None = None + created_at: AwareDatetime | None = None + origin_server: StrictStr | None = None + + @field_validator("created_by", "created_by_email", "origin_server", mode="before") + @classmethod + def blank_means_absent(cls, value): + """ + Treat a blank string in [meta] as "not supplied". + + The backup side writes these fields unconditionally, so an archive made + by a user with no email address on file carries + ``created_by_email = ""``. Since none of this metadata is required (or + even trustworthy), refusing to restore such an archive would be far + worse than not knowing who made it. + """ + if isinstance(value, str) and not value.strip(): + return None + return value + class LearningPackageInputData(InputData): """ @@ -142,8 +159,8 @@ class LearningPackageInputData(InputData): ), ] description: StrictStr | None = Field(default="", max_length=10_000) - created: AwareDatetime | None - updated: AwareDatetime | None + created: AwareDatetime | None = None + updated: AwareDatetime | None = None class DraftInputData(InputData): @@ -155,6 +172,8 @@ class PublishedInputData(InputData): class EntityInputData(InputData): + """A PublishableEntity: either a Component or a Container.""" + can_stand_alone: bool = True # key: str @@ -178,13 +197,22 @@ class EntityInputData(InputData): class SectionInputData(InputData): - section: dict = {} + """Marks an entity as a Section.""" + + # Note: this field is intentionally required, with no default. These three + # container models are all `extra="allow"`, so if the discriminating field + # were optional, every one of them would happily validate every dict and the + # union in EntityInputData.container would always resolve to whichever model + # is listed first. + section: dict + class SubsectionInputData(InputData): - subsection: dict = {} + subsection: dict # Required. See the note on SectionInputData. + class UnitInputData(InputData): - unit: dict = {} + unit: dict # Required. See the note on SectionInputData. class VersionInput(InputData): @@ -202,14 +230,9 @@ class ContainerVersionInput(InputData): children: list[Annotated[str, REF_CONSTRAINTS]] -class VersionsInput(InputData): - versions: dict[ - Annotated[int, Field(gt=0)], - VersionInput, - ] - - class CollectionInput(InputData): + """A named grouping of PublishableEntities within a Learning Package.""" + title: StrictStr = Field(min_length=1) key: Annotated[ str, @@ -225,24 +248,24 @@ class CollectionInput(InputData): ), ] description: StrictStr | None = Field(default="", max_length=10_000) - created: AwareDatetime | None # It looks like we weren't actually serializing the modified date. - created: AwareDatetime | None + created: AwareDatetime | None = None + + # The PublishableEntities that belong to this Collection. Entity refs that + # aren't in the archive are ignored at load time rather than being an error, + # since a Collection with a dangling member is still perfectly usable. + entities: list[Annotated[str, REF_CONSTRAINTS]] = [] # This is the source file where this Collection was defined. This is only # for being able to create useful error messages. We should never be reading # from this file directly because the exact format of this file should be # free to change as needed. That's the responsibilty of the payload.py # module. - src_path: Path | None - - -##### We're not actually using anything below this line yet. - - + src_path: Path | None = None +# --- Output models. Not in use yet; the backup side still writes TOML directly. --- class PackageConfigOutputData(BaseModel): @@ -253,14 +276,6 @@ class PackageConfigOutputData(BaseModel): learning_package: LearningPackageOutputData -class PackageConfigInputData(BaseModel): - """ - Reads the package.toml file when we're reading from a backup archive. - """ - meta: MetaInputData - learning_package: LearningPackageInputData - - class MetaOutputData(BaseModel): """ Output Package Metadata @@ -279,9 +294,6 @@ class MetaOutputData(BaseModel): origin_server: StrictStr - - - class LearningPackageOutputData(BaseModel): """ High level data for a Learning Package. @@ -298,5 +310,3 @@ class LearningPackageOutputData(BaseModel): description: StrictStr created: AwareDatetime updated: AwareDatetime - - diff --git a/src/openedx_content/applets/backup_restore/serializers.py b/src/openedx_content/applets/backup_restore/serializers.py deleted file mode 100644 index 105c25094..000000000 --- a/src/openedx_content/applets/backup_restore/serializers.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -The serializers module for restoration of authoring data. - -Please note that the serializers are defined from the perspective of the -TOML format, with the models as the "source". That is, when the model fields -and TOML fields differ, we'll declare it like this: - - my_toml_field = serializers.BlahField(source="my_model_field") -""" -from datetime import timezone - -from rest_framework import serializers - -from ..components import api as components_api -from ..components.models import ComponentType - - -class LearningPackageSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for learning packages. - - Note: - The ref/key field is serialized but is generally not trustworthy for - restoration. During restore, a new ref may be generated or overridden. - """ - - title = serializers.CharField(required=True) - # The model field is now LearningPackage.package_ref, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="package_ref") - description = serializers.CharField(required=True, allow_blank=True) - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - - -class LearningPackageMetadataSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for learning package metadata. - - Note: - This serializer handles data exported to an archive (e.g., during backup), - but the metadata is not restored to the database and is meant solely for inspection. - """ - format_version = serializers.IntegerField(required=True) - created_by = serializers.CharField(required=False, allow_null=True) - created_by_email = serializers.EmailField(required=False, allow_null=True) - created_at = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - origin_server = serializers.CharField(required=False, allow_null=True) - - -class EntitySerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for publishable entities. - """ - - can_stand_alone = serializers.BooleanField(required=True) - # The model field is now PublishableEntity.entity_ref, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="entity_ref") - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - - -class EntityVersionSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for publishable entity versions. - """ - # We allow_blank because empty unit titles are legal and common. - title = serializers.CharField(required=True, allow_blank=True) - - created = serializers.DateTimeField(required=True, default_timezone=timezone.utc) - version_num = serializers.IntegerField(required=True) - - # Note: Unlike the fields above, `entity_ref` does not appear on the model - # nor in the TOML. This is just added by the validation pipeline for convenience. - entity_ref = serializers.CharField(required=True) - - -class ComponentSerializer(EntitySerializer): # pylint: disable=abstract-method - """ - Serializer for components. - Contains logic to convert entity_key to component_type and component_code. - """ - - def validate(self, attrs): - """ - Custom validation logic: - parse the entity_key into (component_type, component_code). - """ - entity_key = attrs["entity_ref"] - try: - component_type_obj, component_code = _get_or_create_component_type_by_entity_key(entity_key) - attrs["component_type"] = component_type_obj - attrs["component_code"] = component_code - except ValueError as exc: - raise serializers.ValidationError({"key": str(exc)}) - return attrs - - -def _get_or_create_component_type_by_entity_key(entity_key: str) -> tuple[ComponentType, str]: - """ - Get or create a ComponentType based on a full [entity].key string. - - The entity key is expected to be in the format - ``"{namespace}:{type_name}:{component_code}"``. This function will parse out - the ``namespace`` and ``type_name`` parts and use those to get or create the - ComponentType. - - Raises ValueError if the entity_key is not in the expected format. - - Historical note: In Ulmo, this function was part of the public API. This was - inappropriate because the exact format of entity_keys is just a convention - rather than something API callers should count on. That said, it is safe to - assume that in all "v1" archives, the components' entity keys are safe to - parse into (namespace, type, code). So, we have moved this parsing logic - from the public API to just this internal halper function. Future devs, - please do not make new external guarantees about the format of entity keys - (aka entity_refs). A future "v2" backup-restore format will drop this - assumption of parse-ability.. - """ - try: - namespace, type_name, component_code = entity_key.split(':', 2) - except ValueError as exc: - raise ValueError( - f"Invalid entity_key format: {entity_key!r}. " - "Expected format: '{namespace}:{type_name}:{component_code}'" - ) from exc - return components_api.get_or_create_component_type(namespace, type_name), component_code - - -class ComponentVersionSerializer(EntityVersionSerializer): # pylint: disable=abstract-method - """ - Serializer for component versions. - """ - - -class ContainerSerializer(EntitySerializer): # pylint: disable=abstract-method - """ - Serializer for containers. - """ - container = serializers.DictField(required=True) - - def validate_container(self, value): - """ - Custom validation logic for the container field. - Ensures that the container dict has exactly one key which is one of - "section", "subsection", or "unit" values. - """ - errors = [] - if not isinstance(value, dict) or len(value) != 1: - errors.append("Container must be a dict with exactly one key.") - if len(value) == 1: # Only check the key if there is exactly one - container_type = list(value.keys())[0] - if container_type not in ("section", "subsection", "unit"): - errors.append(f"Invalid container value: {container_type}") - if errors: - raise serializers.ValidationError(errors) - return value - - def validate(self, attrs): - """ - Custom validation logic: - parse the container dict to extract the container type. - """ - container = attrs["container"] - container_type = list(container.keys())[0] # It is safe to do this after validate_container - attrs["container_type"] = container_type - attrs.pop("container") # Remove the container field after processing - return attrs - - -class ContainerVersionSerializer(EntityVersionSerializer): # pylint: disable=abstract-method - """ - Serializer for container versions. - """ - container = serializers.DictField(required=True) - - def validate_container(self, value): - """ - Custom validation logic for the container field. - Ensures that the container dict has exactly one key "children" which is a list of strings. - """ - errors = [] - if not isinstance(value, dict) or len(value) != 1: - errors.append("Container must be a dict with exactly one key.") - if "children" not in value: - errors.append("Container must have a 'children' key.") - if "children" in value and not isinstance(value["children"], list): - errors.append("'children' must be a list.") - if errors: - raise serializers.ValidationError(errors) - return value - - def validate(self, attrs): - """ - Custom validation logic: - parse the container dict to extract the children list. - """ - children = attrs["container"]["children"] # It is safe to do this after validate_container - attrs["children"] = children - attrs.pop("container") # Remove the container field after processing - return attrs - - -class CollectionSerializer(serializers.Serializer): # pylint: disable=abstract-method - """ - Serializer for collections. - """ - title = serializers.CharField(required=True) - # The model field is now Collection.collection_code, but the archive format - # still uses "key". A future v2 format may align the name. - key = serializers.CharField(required=True, source="collection_code") - description = serializers.CharField(required=True, allow_blank=True) - entities = serializers.ListField( - child=serializers.CharField(), - required=True, - allow_empty=True, - ) diff --git a/src/openedx_content/applets/backup_restore/toml.py b/src/openedx_content/applets/backup_restore/toml.py index b5afa6174..e1fcff3bd 100644 --- a/src/openedx_content/applets/backup_restore/toml.py +++ b/src/openedx_content/applets/backup_restore/toml.py @@ -3,7 +3,6 @@ """ from datetime import datetime -from typing import Any, Dict import tomlkit from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user @@ -233,27 +232,3 @@ def toml_collection(collection: Collection, entity_refs: list[str]) -> str: doc.add("collection", collection_table) return tomlkit.dumps(doc) - - -def parse_learning_package_toml(content: str) -> dict: - """ - Parse the learning package TOML content and return a dict of its fields. - """ - lp_data: Dict[str, Any] = tomlkit.parse(content) - return lp_data - - -def parse_publishable_entity_toml(content: str) -> dict: - """ - Parse the publishable entity TOML file and return a dict of its fields. - """ - pe_data: Dict[str, Any] = tomlkit.parse(content) - return pe_data - - -def parse_collection_toml(content: str) -> dict: - """ - Parse the collection TOML content and return a dict of its fields. - """ - collection_data: Dict[str, Any] = tomlkit.parse(content) - return collection_data diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py index c86063197..7908682f2 100644 --- a/src/openedx_content/applets/backup_restore/validation.py +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -1,39 +1,222 @@ """ -This is an archive-agnostic validation of the data models. I might actually just -move this to api.py, since most of this work will be done in schema.py +Archive-agnostic validation of the input data models. + +This sits between ``payload.py`` (which knows about files) and ``loading.py`` +(which knows about the database). It answers one question: is this input good +enough to load? It never raises -- every problem it finds is collected onto +``ValidatedLearningPackageInput.errors`` so that someone repairing an archive by +hand can see everything that's wrong in one pass. + +There are three sources of errors here: + +1. Errors that ``payload.py`` already found while reading the files. Those are + ``ExtractionError`` instances and are carried through as-is. +2. Schema errors, from running the raw data through ``CompletePackageInputData``. + Pydantic reports these against a path into the assembled document, so we + translate that back into the archive file it came from. +3. Consistency errors -- cross-references that pydantic has no way to express, + like a container listing a child that isn't anywhere in the archive. """ -import attrs +from __future__ import annotations -from pydantic_core import InitErrorDetails +import attrs from fsspec import AbstractFileSystem +from pydantic import ValidationError +from .errors import ( + BackupRestoreError, + DuplicateVersionError, + MalformedRefError, + MissingVersionError, + SchemaError, + UnknownContainerTypeError, + UnresolvedChildError, +) +from .payload import ROOT_PACKAGE_PATH, UnvalidatedLearningPackageInput from .schema import CompletePackageInputData -from .payload import UnvalidatedLearningPackageInput + @attrs.define(frozen=True) class ValidatedLearningPackageInput: - data: CompletePackageInputData | None # None if it's too broken + """ + The result of validating an archive's contents. - fs: AbstractFileSystem + ``data`` is ``None`` when the input was too broken to build a model from at + all. ``errors`` being non-empty means the restore must not proceed, even if + ``data`` is populated -- a consistency error can be found on a document that + is otherwise structurally valid. + """ - # All these names are terrible. + data: CompletePackageInputData | None + + fs: AbstractFileSystem - # These are the errors that mean this is actually malformed, i.e. JSON - # Schema level validation. - structural_errors: list[InitErrorDetails] + errors: list[BackupRestoreError] - deeper_errors: list # This is stuff we have to dig deeper for, e.g. missing parent-child relationship def validate( unvalidated_lp: UnvalidatedLearningPackageInput, ) -> ValidatedLearningPackageInput: - """ """ - validated = CompletePackageInputData.model_validate(unvalidated_lp.raw_data) - # pretty_print(validated) + """ + Validate extracted archive data, gathering every error we can find. + """ + # Extraction errors are already BackupRestoreErrors, so they just come along. + errors: list[BackupRestoreError] = list(unvalidated_lp.errors) + + try: + data = CompletePackageInputData.model_validate(unvalidated_lp.raw_data) + except ValidationError as val_err: + data = None + errors.extend(_schema_errors_for(val_err, unvalidated_lp)) + + if data is not None: + errors.extend(_consistency_errors_for(data, unvalidated_lp)) return ValidatedLearningPackageInput( - data=validated, + data=data, fs=unvalidated_lp.fs, - structural_errors=[], - deeper_errors=[], - ) \ No newline at end of file + errors=errors, + ) + + +def _schema_errors_for( + val_err: ValidationError, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> list[SchemaError]: + """ + Turn one pydantic ValidationError into one SchemaError per problem found. + """ + return [ + SchemaError( + message=entry["msg"], + **_source_for_loc(entry["loc"], unvalidated_lp), + ) + for entry in val_err.errors() + ] + + +def _source_for_loc( + loc: tuple, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> dict: + """ + Map a pydantic ``loc`` back to the archive file it came from. + + Pydantic reports errors against the combined document we assemble in + ``payload.py``, e.g. ``("entities", "unit1-b7eafb", "versions", 0, "title")``. + Nobody editing an archive has ever seen that document, so we split the ``loc`` + into the file it came from and the location within that file. + """ + match loc: + case ("entities", str() as entity_ref, *rest): + path = unvalidated_lp.entity_path_mapping.get(entity_ref) + # Fall back to naming the entity if we somehow have no path for it. + return {"path": path or f"entities/{entity_ref}", "location": tuple(rest)} + case ("collections", int() as index, *rest): + return { + "path": _collection_path_at(index, unvalidated_lp), + "location": tuple(rest), + } + case ("meta" | "learning_package", *_): + return {"path": ROOT_PACKAGE_PATH, "location": tuple(loc)} + + return {"path": None, "location": tuple(loc)} + + +def _collection_path_at( + index: int, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> str | None: + """ + Look up the source file of the collection at ``index`` in the raw data. + + We can't read this off the validated model, because we only need it when + validation has already failed. + """ + raw_collections = unvalidated_lp.raw_data.get("collections", []) + if 0 <= index < len(raw_collections): + raw_collection = raw_collections[index] + if isinstance(raw_collection, dict): + return raw_collection.get("src_path") + return None + + +def _consistency_errors_for( + data: CompletePackageInputData, + unvalidated_lp: UnvalidatedLearningPackageInput, +) -> list[BackupRestoreError]: + """ + Check the cross-references that pydantic can't express. + + Every check here corresponds to something that would otherwise blow up in the + middle of ``loading.py``, with a traceback pointing at our code instead of at + the part of the archive that's actually wrong. + """ + errors: list[BackupRestoreError] = [] + known_refs = set(data.entities) + + def path_for(entity_ref: str) -> str | None: + return unvalidated_lp.entity_path_mapping.get(entity_ref) + + for entity_ref, entity in sorted(data.entities.items()): + path = path_for(entity_ref) + + # Check: is this a container type we actually know how to build? + if isinstance(entity.container, dict): + declared = ", ".join(sorted(entity.container)) or "(empty)" + errors.append( + UnknownContainerTypeError( + f'Entity "{entity_ref}" declares an unsupported container ' + f"type: {declared}", + path=path, + ) + ) + elif entity.container is None: + # Not a container, so it's a Component, and we derive the component + # type from the ref itself. + if len(entity_ref.split(":")) != 3: + errors.append( + MalformedRefError( + f'Component ref "{entity_ref}" should be of the form ' + '"{namespace}:{type}:{code}"', + path=path, + ) + ) + + # Check: no version_num declared twice for the same entity. + version_nums = [version.version_num for version in entity.versions] + for duplicated in sorted({v for v in version_nums if version_nums.count(v) > 1}): + errors.append( + DuplicateVersionError( + f'Entity "{entity_ref}" declares version {duplicated} more than once', + path=path, + ) + ) + + # Check: the draft/published pointers name versions that exist here. + available = set(version_nums) + for label, pointer in (("draft", entity.draft), ("published", entity.published)): + if pointer.version_num is not None and pointer.version_num not in available: + errors.append( + MissingVersionError( + f'Entity "{entity_ref}" points [entity.{label}] at version ' + f"{pointer.version_num}, which is not in the archive", + path=path, + ) + ) + + # Check: every child of every container version is in the archive. + for version in entity.versions: + if version.container is None: + continue + for child_ref in version.container.children: + if child_ref not in known_refs: + errors.append( + UnresolvedChildError( + f'Entity "{entity_ref}" v{version.version_num} lists child ' + f'"{child_ref}", which is not defined in the archive', + path=path, + ) + ) + + return errors diff --git a/src/openedx_content/applets/backup_restore/zipper.py b/src/openedx_content/applets/backup_restore/zipper.py index 321cef3e2..b70694e1a 100644 --- a/src/openedx_content/applets/backup_restore/zipper.py +++ b/src/openedx_content/applets/backup_restore/zipper.py @@ -3,25 +3,17 @@ including a TOML representation of the learning package and its entities. """ import hashlib -import time -import tomllib import zipfile -from collections import defaultdict -from dataclasses import asdict, dataclass from datetime import datetime, timezone -from io import StringIO from pathlib import Path -from typing import Any, List, Literal, Optional, Tuple +from typing import List, Optional, Tuple from django.contrib.auth.models import User as UserType # pylint: disable=imported-auth-user -from django.db import transaction from django.db.models import Prefetch, QuerySet from django.utils.text import slugify -from rest_framework import serializers from openedx_content.models_api import ( Collection, - ComponentType, ComponentVersion, ComponentVersionMedia, LearningPackage, @@ -31,33 +23,10 @@ ) from ..collections import api as collections_api -from ..components import api as components_api -from ..containers import api as containers_api -from ..media import api as media_api from ..publishing import api as publishing_api -from ..sections.models import Section -from ..subsections.models import Subsection -from ..units.models import Unit -from .serializers import ( - CollectionSerializer, - ComponentSerializer, - ComponentVersionSerializer, - ContainerSerializer, - ContainerVersionSerializer, - LearningPackageMetadataSerializer, - LearningPackageSerializer, -) -from .toml import ( - parse_collection_toml, - parse_learning_package_toml, - parse_publishable_entity_toml, - toml_collection, - toml_learning_package, - toml_publishable_entity, -) +from .toml import toml_collection, toml_learning_package, toml_publishable_entity TOML_PACKAGE_NAME = "package.toml" -DEFAULT_USERNAME = "command" def slugify_hashed_filename(identifier: str) -> str: @@ -410,730 +379,3 @@ def create_zip(self, path: str) -> None: toml_collection(collection, list(entity_refs_related)), timestamp=collection.modified, ) - - -@dataclass -class RestoreLearningPackageData: - """ - Data about the restored learning package. - """ - id: int # The ID of the restored learning package - package_ref: str # The package_ref of the restored learning package (may be different if staged) - archive_package_ref: str # The original package_ref from the archive - archive_org_code: str | None # The org code parsed from archive_package_ref, or None if unparseable - archive_package_code: str | None # The package code parsed from archive_package_ref, or None if unparseable - title: str - num_containers: int - num_sections: int - num_subsections: int - num_units: int - num_components: int - num_collections: int - - -@dataclass -class BackupMetadata: - """ - Metadata about the backup operation. - """ - format_version: int - created_at: str - created_by: str | None = None - created_by_email: str | None = None - original_server: str | None = None - - -@dataclass -class RestoreResult: - """ - Result of the restore operation. - """ - status: Literal["success", "error"] - log_file_error: StringIO | None = None - lp_restored_data: RestoreLearningPackageData | None = None - backup_metadata: BackupMetadata | None = None - - -def unpack_package_ref(package_ref: str) -> tuple[str | None, str | None]: - """ - Try to parse org_code and package_code from a package_ref. - - By convention, package_refs take the form ``"{prefix}:{org_code}:{package_code}"``, - but this is only a convention — package_ref is opaque and the parse may fail. - Returns ``(None, None)`` if the ref does not match the expected format. - """ - parts = package_ref.split(":") - if len(parts) < 3: - return None, None - _, org_code, package_code = parts[:3] - return org_code, package_code - - -def generate_staged_package_ref(archive_package_ref: str, user: UserType) -> str: - """ - Generate a staged learning package ref based on the archive's package_ref. - - We can't trust package_ref from the archive directly, because the archive - could specify *any* arbitrary package_ref, and the user may or may not be - permitted to create an Package using that ref. So, instead, this function - generates a unique and semi-human-readable package_ref which is namespaced - to the current user and appropriate to provisionally save the package under. - The package_ref from the archive can then be presented to the user as a - *suggestion*, which they may or may not choose to use. - - Please note that the ref returned by this function is valid for Packages is - a generic sense, but it's not a valid Content Library key. Callers who are - restoring a Package for Library usage will need to replace this staged - package_ref before being able to render the Library's content. - - Arguments: - archive_package_ref (str): The original package_ref from the archive. - user (UserType | None): The user performing the restore operation. - - Example: - Input: "lib:WGU:LIB_C001" - Output: "lp-restore:dave:WGU:LIB_C001:1728575321" - """ - username = user.username - org_code, package_code = unpack_package_ref(archive_package_ref) - timestamp = int(time.time() * 1000) # Current time in milliseconds - if org_code and package_code: - return f"lp-restore:{username}:{org_code}:{package_code}:{timestamp}" - # Fallback for non-conventional package_refs - return f"lp-restore:{username}:{archive_package_ref}:{timestamp}" - - -class LearningPackageUnzipper: - """ - Handles extraction and restoration of learning package data from a zip archive. - - Args: - zipf (zipfile.ZipFile): The zip file containing the learning package data. - user (UserType | None): The user performing the restore operation. Not necessarily the creator. - generate_new_key (bool): Whether to generate a new key for the restored learning package. - - Returns: - dict[str, Any]: The result of the restore operation, including any errors encountered. - - Responsibilities: - - Parse and organize files from the zip structure. - - Restore learning package, containers, components, and collections to the database. - - Ensure atomicity of the restore process. - - Usage: - unzipper = LearningPackageUnzipper(zip_file) - result = unzipper.load() - """ - - def __init__(self, zipf: zipfile.ZipFile, package_ref: str | None = None, user: UserType | None = None): - self.zipf = zipf - self.user = user - self.user_id = getattr(self.user, "id", None) - self.package_ref = package_ref # If provided, use this package_ref for the restored learning package - self.learning_package_id: LearningPackage.ID | None = None # Will be set upon restoration - self.utc_now: datetime = datetime.now(timezone.utc) - self.component_types_cache: dict[tuple[str, str], ComponentType] = {} - self.errors: list[dict[str, Any]] = [] - # Maps for resolving relationships - self.components_map_by_ref: dict[str, Any] = {} - self.units_map_by_ref: dict[str, Any] = {} - self.subsections_map_by_ref: dict[str, Any] = {} - self.sections_map_by_ref: dict[str, Any] = {} - self.all_publishable_entity_refs: set[str] = set() - self.all_published_entities_versions: set[tuple[str, int]] = set() # To track published entity versions - - # -------------------------- - # Public API - # -------------------------- - - @transaction.atomic - def load(self) -> dict[str, Any]: - """Extracts and restores all objects from the ZIP archive in an atomic transaction.""" - - # Step 1: Validate presence of package.toml and basic structure - _, organized_files = self.check_mandatory_files() - if self.errors: - # Early return if preliminary checks fail since mandatory files are missing - result = RestoreResult( - status="error", - log_file_error=self._write_errors(), # return a StringIO with the errors - lp_restored_data=None, - backup_metadata=None, - ) - return asdict(result) - - # Step 2: Extract and validate learning package, entities and collections - # Errors are collected and reported at the end - # No saving to DB happens until all validations pass - learning_package_validated = self._extract_learning_package(organized_files["learning_package"]) - lp_metadata = learning_package_validated.pop("metadata", {}) - - components_validated = self._extract_entities( - organized_files["components"], ComponentSerializer, ComponentVersionSerializer - ) - containers_validated = self._extract_entities( - organized_files["containers"], ContainerSerializer, ContainerVersionSerializer - ) - - collections_validated = self._extract_collections( - organized_files["collections"] - ) - - # Step 3.1: If there are validation errors, return them without saving anything - if self.errors: - result = RestoreResult( - status="error", - log_file_error=self._write_errors(), # return a StringIO with the errors - lp_restored_data=None, - backup_metadata=None, - ) - return asdict(result) - - # Step 3.2: Save everything to the DB - # All validations passed, we can proceed to save everything - # Save the learning package first to get its ID - archive_package_ref = learning_package_validated["package_ref"] - learning_package = self._save( - learning_package_validated, - components_validated, - containers_validated, - collections_validated, - component_static_files=organized_files["component_static_files"] - ) - - num_containers = sum( - len(containers_validated.get(container_type, [])) - for container_type in ["section", "subsection", "unit"] - ) - - org_code, package_code = unpack_package_ref(archive_package_ref) - result = RestoreResult( - status="success", - log_file_error=None, - lp_restored_data=RestoreLearningPackageData( - id=learning_package.id, - package_ref=learning_package.package_ref, - archive_package_ref=archive_package_ref, - archive_org_code=org_code, - archive_package_code=package_code, - title=learning_package.title, - num_containers=num_containers, - num_sections=len(containers_validated.get("section", [])), - num_subsections=len(containers_validated.get("subsection", [])), - num_units=len(containers_validated.get("unit", [])), - num_components=len(components_validated["components"]), - num_collections=len(collections_validated["collections"]), - ), - backup_metadata=BackupMetadata( - format_version=lp_metadata.get("format_version", 1), - created_by=lp_metadata.get("created_by"), - created_by_email=lp_metadata.get("created_by_email"), - created_at=lp_metadata.get("created_at"), - original_server=lp_metadata.get("origin_server"), - ) if lp_metadata else None, - ) - return asdict(result) - - def check_mandatory_files(self) -> Tuple[list[dict[str, Any]], dict[str, Any]]: - """ - Check for the presence of mandatory files in the zip archive. - So far, the only mandatory file is package.toml. - """ - organized_files = self._get_organized_file_list(self.zipf.namelist()) - - if not organized_files["learning_package"]: - self.errors.append({"file": TOML_PACKAGE_NAME, "errors": "Missing learning package file."}) - - return self.errors, organized_files - - # -------------------------- - # Extract + Validate - # -------------------------- - - def _extract_learning_package(self, package_file: str) -> dict[str, Any]: - """Extract and validate the learning package TOML file.""" - toml_content_text = self._read_file_from_zip(package_file) - toml_content_dict = parse_learning_package_toml(toml_content_text) - lp = toml_content_dict.get("learning_package") - lp_metadata = toml_content_dict.get("meta") - - # Validate learning package data - lp_serializer = LearningPackageSerializer(data=lp) - if not lp_serializer.is_valid(): - self.errors.append({"file": f"{package_file} learning package section", "errors": lp_serializer.errors}) - - # Validate metadata if present - lp_metadata_serializer = LearningPackageMetadataSerializer(data=lp_metadata) - if not lp_metadata_serializer.is_valid(): - self.errors.append({"file": f"{package_file} meta section", "errors": lp_metadata_serializer.errors}) - - lp_validated = lp_serializer.validated_data if lp_serializer.is_valid() else {} - lp_metadata = lp_metadata_serializer.validated_data if lp_metadata_serializer.is_valid() else {} - lp_validated["metadata"] = lp_metadata - return lp_validated - - def _extract_entities( - self, - entity_files: list[str], - entity_serializer: type[serializers.Serializer], - version_serializer: type[serializers.Serializer], - ) -> dict[str, Any]: - """Generic extraction + validation pipeline for containers or components.""" - results: dict[str, list[Any]] = defaultdict(list) - - for file in entity_files: - if not file.endswith(".toml"): - # Skip non-TOML files - continue - - entity_data, draft_version, published_version = self._load_entity_data(file) - serializer = entity_serializer( - data={"created": self.utc_now, "created_by": None, **entity_data} - ) - - if not serializer.is_valid(): - self.errors.append({"file": file, "errors": serializer.errors}) - continue - - entity_data = serializer.validated_data - self.all_publishable_entity_refs.add(entity_data["entity_ref"]) - entity_type = entity_data.pop("container_type", "components") - results[entity_type].append(entity_data) - - valid_versions = self._validate_versions( - entity_data, - draft_version, - published_version, - version_serializer, - file=file - ) - if valid_versions["draft"]: - results[f"{entity_type}_drafts"].append(valid_versions["draft"]) - if valid_versions["published"]: - results[f"{entity_type}_published"].append(valid_versions["published"]) - - return results - - def _extract_collections( - self, - collection_files: list[str], - ) -> dict[str, Any]: - """Extraction + validation pipeline for collections.""" - results: dict[str, list[Any]] = defaultdict(list) - - for file in collection_files: - if not file.endswith(".toml"): - # Skip non-TOML files - continue - toml_content = self._read_file_from_zip(file) - collection_data = parse_collection_toml(toml_content) - collection_data = collection_data.get("collection", {}) - serializer = CollectionSerializer(data={"created_by": None, **collection_data}) - if not serializer.is_valid(): - self.errors.append({"file": f"{file} collection section", "errors": serializer.errors}) - continue - collection_validated = serializer.validated_data - entities_list = collection_validated["entities"] - for entity_ref in entities_list: - if entity_ref not in self.all_publishable_entity_refs: - self.errors.append({ - "file": file, - "errors": f"Entity ref {entity_ref} not found for collection {collection_validated.get('key')}" - }) - results["collections"].append(collection_validated) - - return results - - # -------------------------- - # Save Logic - # -------------------------- - - def _save( - self, - learning_package: dict[str, Any], - components: dict[str, Any], - containers: dict[str, Any], - collections: dict[str, Any], - *, - component_static_files: dict[str, List[str]] - ) -> LearningPackage: - """Persist all validated entities in two phases: published then drafts.""" - - # Important: If not using a specific LP ref/key, generate a temporary one - # We cannot use the original key because it may generate security issues - if not self.package_ref: - # Generate a tmp ref for the staged learning package - if not self.user: - raise ValueError("User is required to generate a staged package_ref") - learning_package["package_ref"] = generate_staged_package_ref( - archive_package_ref=learning_package["package_ref"], - user=self.user - ) - else: - learning_package["package_ref"] = self.package_ref - - learning_package_obj = publishing_api.create_learning_package(**learning_package) - self.learning_package_id = learning_package_obj.id - - with publishing_api.bulk_draft_changes_for(learning_package_obj.id): - self._save_components(learning_package_obj, components, component_static_files) - self._save_units(learning_package_obj, containers) - self._save_subsections(learning_package_obj, containers) - self._save_sections(learning_package_obj, containers) - self._save_collections(learning_package_obj, collections) - publishing_api.publish_all_drafts(learning_package_obj.id) - - with publishing_api.bulk_draft_changes_for(learning_package_obj.id): - self._save_draft_versions(components, containers, component_static_files) - - return learning_package_obj - - def _save_collections(self, learning_package, collections): - """Save collections and their entities.""" - for valid_collection in collections.get("collections", []): - entities = valid_collection.pop("entities", []) - collection = collections_api.create_collection( - learning_package.id, created_by=self.user_id, **valid_collection - ) - collection = collections_api.add_to_collection( - learning_package_id=learning_package.id, - collection_code=collection.collection_code, - entities_qset=publishing_api.get_publishable_entities(learning_package.id).filter( - entity_ref__in=entities - ) - ) - - def _save_components(self, learning_package, components, component_static_files): - """Save components and published component versions.""" - for valid_component in components.get("components", []): - entity_ref = valid_component.pop("entity_ref") - component = components_api.create_component(learning_package.id, created_by=self.user_id, **valid_component) - self.components_map_by_ref[entity_ref] = component - - for valid_published in components.get("components_published", []): - entity_ref = valid_published.pop("entity_ref") - version_num = valid_published["version_num"] # Should exist, validated earlier - component = self.components_map_by_ref[entity_ref] - media_to_replace = self._resolve_static_files( - version_num, entity_ref, component.component_type, component_static_files - ) - self.all_published_entities_versions.add( - (entity_ref, version_num) - ) # Track published version - components_api.create_next_component_version( - component.publishable_entity.id, - media_to_replace=media_to_replace, - force_version_num=valid_published.pop("version_num", None), - created_by=self.user_id, - **valid_published - ) - - def _save_container( - self, - learning_package, - containers, - *, - container_cls: containers_api.ContainerSubclass, - container_map: dict, - children_map: dict, - ): - """Internal logic for _save_units, _save_subsections, and _save_sections""" - type_code = container_cls.type_code # e.g. "unit" - for data in containers.get(type_code, []): - entity_ref = data.pop("entity_ref") - container = containers_api.create_container( - learning_package.id, - # As of Verawood, the primary identity of a container is its - # `container_code`. By convention, this equals the `entity_ref` - # (aka `[entity].key`). It's safe to assume that all v1 - # archives have an identical `entity_ref` and `container_code` for each - # entity-container. BUT, this assumpion may not hold true v2+. - container_code=entity_ref, - **data, # should this be allowed to override any of the following fields? - created_by=self.user_id, - container_cls=container_cls, - ) - container_map[entity_ref] = container # e.g. `self.units_map_by_ref[entity_ref] = unit` - - for valid_published in containers.get(f"{type_code}_published", []): - entity_ref = valid_published.pop("entity_ref") - children = self._resolve_children(valid_published, children_map) - version_num = valid_published.pop("version_num", None) - self.all_published_entities_versions.add((entity_ref, version_num)) - containers_api.create_next_container_version( - container_map[entity_ref], - force_version_num=version_num, - **valid_published, # should this be allowed to override any of the following fields? - entities=children, - created_by=self.user_id, - ) - - def _save_units(self, learning_package, containers): - """Save units and published unit versions.""" - self._save_container( - learning_package, - containers, - container_cls=Unit, - container_map=self.units_map_by_ref, - children_map=self.components_map_by_ref, - ) - - def _save_subsections(self, learning_package, containers): - """Save subsections and published subsection versions.""" - self._save_container( - learning_package, - containers, - container_cls=Subsection, - container_map=self.subsections_map_by_ref, - children_map=self.units_map_by_ref, - ) - - def _save_sections(self, learning_package, containers): - """Save sections and published section versions.""" - self._save_container( - learning_package, - containers, - container_cls=Section, - container_map=self.sections_map_by_ref, - children_map=self.subsections_map_by_ref, - ) - - def _save_draft_versions(self, components, containers, component_static_files): - """Save draft versions for all entity types.""" - for valid_draft in components.get("components_drafts", []): - entity_ref = valid_draft.pop("entity_ref") - version_num = valid_draft["version_num"] # Should exist, validated earlier - if self._is_version_already_exists(entity_ref, version_num): - continue - component = self.components_map_by_ref[entity_ref] - media_to_replace = self._resolve_static_files( - version_num, entity_ref, component.component_type, component_static_files - ) - components_api.create_next_component_version( - component.publishable_entity.id, - media_to_replace=media_to_replace, - force_version_num=valid_draft.pop("version_num", None), - # Drafts can diverge from published, so we allow ignoring previous media - # Use case: published v1 had files A, B; draft v2 only has file A - ignore_previous_media=True, - created_by=self.user_id, - **valid_draft - ) - - def _process_draft_containers( - container_cls: containers_api.ContainerSubclass, - container_map: dict, - children_map: dict, - ): - for valid_draft in containers.get(f"{container_cls.type_code}_drafts", []): - entity_ref = valid_draft.pop("entity_ref") - version_num = valid_draft["version_num"] # Should exist, validated earlier - if self._is_version_already_exists(entity_ref, version_num): - continue - children = self._resolve_children(valid_draft, children_map) - del valid_draft["version_num"] - containers_api.create_next_container_version( - container_map[entity_ref], - **valid_draft, # should this be allowed to override any of the following fields? - entities=children, - force_version_num=version_num, - created_by=self.user_id, - ) - - _process_draft_containers(Unit, self.units_map_by_ref, children_map=self.components_map_by_ref) - _process_draft_containers(Subsection, self.subsections_map_by_ref, children_map=self.units_map_by_ref) - _process_draft_containers(Section, self.sections_map_by_ref, children_map=self.subsections_map_by_ref) - - # -------------------------- - # Utilities - # -------------------------- - - def _format_errors(self) -> str: - """Return formatted error content as a string.""" - if not self.errors: - return "" - lines = [f"{err['file']}: {err['errors']}" for err in self.errors] - return "Errors encountered during restore:\n" + "\n".join(lines) + "\n" - - def _write_errors(self) -> StringIO | None: - """ - Write errors to a StringIO buffer. - """ - content = self._format_errors() - if not content: - return None - return StringIO(content) - - def _is_version_already_exists(self, entity_ref: str, version_num: int) -> bool: - """ - Check if a version already exists for a given entity_ref and version number. - - Note: - Skip creating draft if this version is already published - Why? Because the version itself is already created and - we don't want to create duplicate versions. - Otherwise, we will raise an IntegrityError on PublishableEntityVersion - due to unique constraints between publishable_entity and version_num. - """ - identifier = (entity_ref, version_num) - return identifier in self.all_published_entities_versions - - def _resolve_static_files( - self, - num_version: int, - entity_ref: str, - component_type: ComponentType, - static_files_map: dict[str, List[str]] - ) -> dict[str, bytes | int]: - """Resolve static file paths into their binary media content.""" - resolved_files: dict[str, bytes | int] = {} - - static_file_key = f"{entity_ref}:v{num_version}" # e.g., "xblock.v1:html:my_component_123456:v1" - block_type = component_type.name # e.g., "html" - static_files = static_files_map.get(static_file_key, []) - for static_file in static_files: - local_key = static_file.split(f"v{num_version}/")[-1] - with self.zipf.open(static_file, "r") as f: - media_bytes = f.read() - if local_key == "block.xml": - # Special handling for block.xml to ensure - # storing the value as a media instance - if not self.learning_package_id: - raise ValueError("learning_package_id must be set before resolving static files.") - text_media = media_api.get_or_create_text_media( - self.learning_package_id, - media_api.get_or_create_media_type(f"application/vnd.openedx.xblock.v1.{block_type}+xml").id, - text=media_bytes.decode("utf-8"), - created=self.utc_now, - ) - resolved_files[local_key] = text_media.id - else: - resolved_files[local_key] = media_bytes - return resolved_files - - def _resolve_children(self, entity_data: dict[str, Any], lookup_map: dict[str, Any]) -> list[Any]: - """Resolve child entity refs into model instances.""" - children_refs = entity_data.pop("children", []) - return [lookup_map[ref] for ref in children_refs if ref in lookup_map] - - def _load_entity_data( - self, entity_file: str - ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]: - """Load entity data and its versions from TOML.""" - entity_toml_txt = self._read_file_from_zip(entity_file) - entity_toml_dict = parse_publishable_entity_toml(entity_toml_txt) - entity_data = entity_toml_dict.get("entity", {}) - version_data = entity_toml_dict.get("version", []) - return entity_data, *self._get_versions_to_write(version_data, entity_data) - - def _validate_versions(self, entity_data, draft, published, serializer_cls, *, file) -> dict[str, Any]: - """Validate draft/published versions with serializer.""" - valid = {"draft": None, "published": None} - for label, version in [("draft", draft), ("published", published)]: - if not version: - continue - serializer = serializer_cls( - data={ - "entity_ref": entity_data["entity_ref"], - "created": self.utc_now, - "created_by": None, - **version - } - ) - if serializer.is_valid(): - valid[label] = serializer.validated_data - else: - self.errors.append({"file": file, "errors": serializer.errors}) - return valid - - def _read_file_from_zip(self, filename: str) -> str: - """Read and decode a UTF-8 file from the zip archive.""" - with self.zipf.open(filename) as f: - return f.read().decode("utf-8") - - def _get_organized_file_list(self, file_paths: list[str]) -> dict[str, Any]: - """Organize file paths into categories: learning_package, containers, components, collections.""" - organized: dict[str, Any] = { - "learning_package": None, - "containers": [], - "components": [], - "component_static_files": defaultdict(list), - "collections": [], - } - - # This is going to map static file directory roots to the appropriate - # entity refs. - comp_paths_to_refs = {} - - # The ordering of the file processing is important because we need to - # ensure that TOML files for a given component are processed before the - # static files for that component. '.' sorts before '/', so "foo.toml" - # will sort before "foo/component_versions/v1/static/figure1.webp" or - # any other subdirectory of the "foo" component. Processing the TOML - # first allows us to map the directory to a entity ref. - for path in sorted(file_paths): - if path.endswith("/"): - # Skip directories - continue - if path == TOML_PACKAGE_NAME: - organized["learning_package"] = path - elif path.startswith("entities/") and str(Path(path).parent) == "entities" and path.endswith(".toml"): - # Top-level entity TOML files are considered containers - organized["containers"].append(path) - elif path.startswith("entities/"): - if path.endswith(".toml"): - # Component entity TOML files - organized["components"].append(path) - component_toml_str = self._read_file_from_zip(path) - component_toml = tomllib.loads(component_toml_str) - entity_ref = component_toml['entity']['key'] - comp_path = path.removesuffix(".toml") - - # This maps the root path of a component, e.g."entities/xblock.v1/html/my_component_a822bb" - # to the actual ref, e.g. "xblock.v1:html:my_component". The last part of the ref will - # often correlate to the directory name, but does not have to (a hash is sometimes added). - comp_paths_to_refs[comp_path] = entity_ref - - else: - # Component static files - # Path structure: entities////component_versions//static/... - # Example: entities/xblock.v1/html/my_component_a822bb/component_versions/v1/static/... - - # e.g. 'entities/xblock.v1/html/my_component_a822bb' - component_root_path = '/'.join(Path(path).parts[0:4]) - - try: - component_ref = comp_paths_to_refs[component_root_path] - except KeyError: - self.errors.append( - { - "file": path, - "errors": f"Missing component TOML file at {component_root_path}.toml" - } - ) - continue - - num_version = Path(path).parts[5] if len(Path(path).parts) > 5 else "v1" # e.g., 'v1' - - component_ref += f":{num_version}" - organized["component_static_files"][component_ref].append(path) - - elif path.startswith("collections/") and path.endswith(".toml"): - # Collection TOML files - organized["collections"].append(path) - - return organized - - def _get_versions_to_write( - self, - version_data: list[dict[str, Any]], - entity_data: dict[str, Any] - ) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]: - """Return the draft and published versions to write, based on entity data.""" - draft_num = entity_data.get("draft", {}).get("version_num") - published_num = entity_data.get("published", {}).get("version_num") - lookup = {v.get("version_num"): v for v in version_data} - return ( - lookup.get(draft_num) if draft_num else None, - lookup.get(published_num) if published_num else None, - ) diff --git a/src/openedx_content/management/commands/encode.py b/src/openedx_content/management/commands/encode.py deleted file mode 100644 index 84171c9fb..000000000 --- a/src/openedx_content/management/commands/encode.py +++ /dev/null @@ -1,98 +0,0 @@ -from datetime import datetime, timezone -import logging - -from django.core.management import CommandError -from django.core.management.base import BaseCommand - -logger = logging.getLogger(__name__) - -import json - - -from pydantic import BaseModel, Field -from pydantic.config import ConfigDict -from pydantic.json_schema import models_json_schema - -from typing import Annotated, Optional - - -class EntityVersion(BaseModel): - version_num: int - title: str - -class VersionRef(BaseModel): - version_num: Optional[int] = None - -class Entity(BaseModel): - can_stand_alone: bool - key: str - created: datetime - draft: VersionRef - published: VersionRef - versions: list[EntityVersion] - -class EntityRoot(BaseModel): - entity: Entity - -from openedx_content.applets.backup_restore.schema import ( - LearningPackageOutputData, PackageConfigOutputData, MetaOutputData -) - -class Command(BaseCommand): - """ - Django management command to export a learning package to a zip file. - """ - help = 'Export a learning package to a zip file.' - - def add_arguments(self, parser): - pass - - def handle(self, *args, **options): - now = datetime.now(tz=timezone.utc) - config = PackageConfigOutputData( - meta=MetaOutputData( - format_version=1, - created_by="dave", - created_by_email="dave@axim.org", - created_at=now, - ), - learning_package=LearningPackageOutputData( - title="Fun Library", - key="lib:Axim:FunLib", - description="", - created=now, - updated=now, - origin_server="studio.local.openedx.io:8001", - ) - ) - toml_output = tomli_w.dumps(config.model_dump(exclude_defaults=False)) - print(toml_output) - - print(json.dumps(PackageConfigOutputData.model_json_schema(), indent=2)) - - - def handle_old(self, *args, **options): - e = Entity( - can_stand_alone=True, - key="xblock.v1:html:hi-there-9d01929cda81", - created=datetime.now(tz=timezone.utc), - draft=VersionRef(version_num=3), - published=VersionRef(version_num=None), - versions = [ - EntityVersion( - version_num=x, - title=f"Title {x}", - ) - for x in range(10) - ], - ) - base = EntityRoot( - entity=e, - ) - - #toml_output = tomli_w.dumps(base.model_dump(exclude_defaults=True)) - #print(toml_output) - - - print(json.dumps(EntityRoot.model_json_schema(), indent=2)) - diff --git a/src/openedx_content/management/commands/lp_load.py b/src/openedx_content/management/commands/lp_load.py index 97fc55870..3ff64e269 100644 --- a/src/openedx_content/management/commands/lp_load.py +++ b/src/openedx_content/management/commands/lp_load.py @@ -9,6 +9,7 @@ from django.core.management.base import BaseCommand from openedx_content.applets.backup_restore.api import load_learning_package +from openedx_content.applets.backup_restore.errors import BackupRestoreError, RestoreFailedError logger = logging.getLogger(__name__) @@ -17,41 +18,52 @@ class Command(BaseCommand): """ - Django management command to load a learning package from a zip file. + Django management command to load a learning package from a backup archive. """ - help = 'Load a learning package from a zip file.' + help = 'Load a learning package from a backup archive (a .zip file or an unzipped directory).' def add_arguments(self, parser): - parser.add_argument('file_name', type=str, help='The path of the input zip file to load.') + parser.add_argument( + 'path', + type=str, + help='The path of the archive to load: either a .zip file or a directory.', + ) parser.add_argument('username', type=str, help='The username of the user performing the load operation.') + parser.add_argument( + '--package-ref', + type=str, + default=None, + help=( + "The package ref to restore under. If omitted, a staged ref " + "namespaced to the user is generated." + ), + ) def handle(self, *args, **options): - file_name = options['file_name'] + path = options['path'] username = options['username'] - if not file_name.lower().endswith(".zip"): - raise CommandError("Input file name must end with .zip") + package_ref = options['package_ref'] + try: - start_time = time.time() - # Get the user performing the operation user = User.objects.get(username=username) + except User.DoesNotExist as exc: + raise CommandError(f"No such user: {username}") from exc + + start_time = time.time() + try: + result = load_learning_package(path, user=user, package_ref=package_ref) + except RestoreFailedError as exc: + # The archive is bad. Show every problem we found, not just the first. + raise CommandError(exc.as_text()) from exc + except BackupRestoreError as exc: + raise CommandError(f"Failed to load '{path}': {exc}") from exc + except Exception as exc: + logger.exception("Failed to load archive %s", path) + raise CommandError(f"Failed to load '{path}': {exc}") from exc - result = load_learning_package(file_name, user=user) - duration = time.time() - start_time - if result["status"] == "error": - message = "Errors encountered during restore:\n" - log_buffer = result.get("log_file_error") - if log_buffer: - message += log_buffer.getvalue() - raise CommandError(message) - message = f'{file_name} loaded successfully (duration: {duration:.2f} seconds)' - self.stdout.write(self.style.SUCCESS(message)) - except FileNotFoundError as exc: - message = f"Learning package file {file_name} not found: {exc}" - raise CommandError(message) from exc - except Exception as e: - message = f"Failed to load '{file_name}': {e}" - logger.exception( - "Failed to load zip file %s ", - file_name, - ) - raise CommandError(message) from e + duration = time.time() - start_time + restored = result.lp_restored_data + self.stdout.write(self.style.SUCCESS( + f'{path} loaded successfully as "{restored.package_ref}" ' + f'(duration: {duration:.2f} seconds)' + )) diff --git a/src/openedx_content/management/commands/lp_load2.py b/src/openedx_content/management/commands/lp_load2.py deleted file mode 100644 index f23592b09..000000000 --- a/src/openedx_content/management/commands/lp_load2.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Django management commands to handle restore learning packages (WIP) -""" -import logging -import time - -from django.contrib.auth import get_user_model -from django.core.management import CommandError -from django.core.management.base import BaseCommand - -from openedx_content.applets.backup_restore.api import load_learning_package - -logger = logging.getLogger(__name__) - -User = get_user_model() - - - -class Command(BaseCommand): - """ - Django management command to load a Learning Package. - """ - help = 'Load a learning package from a zip file.' - - def add_arguments(self, parser): - parser.add_argument('path', type=str, help='The path of the directory or file to load from.') - parser.add_argument('package_ref', type=str, help="Learning Package Ref: often a v2 library key.") - parser.add_argument('username', type=str, help='The username of the user performing the load operation.') - - - def handle(self, *args, **options): - path = options['path'] - package_ref = options['package_ref'] - username = options['username'] - - user = User.objects.get(username=username) - - load_learning_package(path, user=user, package_ref=package_ref) - - return 0 - if not path.lower().endswith(".zip"): - raise CommandError("Input file name must end with .zip") - try: - start_time = time.time() - # Get the user performing the operation - user = User.objects.get(username=username) - - result = load_learning_package(path, user=user) - duration = time.time() - start_time - if result["status"] == "error": - message = "Errors encountered during restore:\n" - log_buffer = result.get("log_file_error") - if log_buffer: - message += log_buffer.getvalue() - raise CommandError(message) - message = f'{path} loaded successfully (duration: {duration:.2f} seconds)' - self.stdout.write(self.style.SUCCESS(message)) - except FileNotFoundError as exc: - message = f"Learning package file {path} not found: {exc}" - raise CommandError(message) from exc - except Exception as e: - message = f"Failed to load '{path}': {e}" - logger.exception( - "Failed to load zip file %s ", - path, - ) - raise CommandError(message) from e diff --git a/test_utils/zip_file_utils.py b/test_utils/zip_file_utils.py index 31dc4c589..64e75a0cb 100644 --- a/test_utils/zip_file_utils.py +++ b/test_utils/zip_file_utils.py @@ -24,3 +24,28 @@ def folder_to_inmemory_zip(folder_path: str) -> zipfile.ZipFile: zipf.write(file_path, arcname=str(arcname)) buffer.seek(0) return zipfile.ZipFile(buffer, "r") + + +def folder_to_zip_path(folder_path: str, dest_dir: str, name: str = "archive.zip") -> str: + """ + Write the contents of a folder out as a real zip file on disk. + + Unlike ``folder_to_inmemory_zip``, this returns a *path*, which is what the + restore pipeline takes (it opens the archive itself, so that it can support + both zip files and plain directories). + + Args: + folder_path (str): Path to the folder to zip. + dest_dir (str): Directory to write the zip file into. + name (str): File name to give the zip file. + + Returns: + str: The path of the zip file that was written. + """ + folder = Path(folder_path) + zip_path = Path(dest_dir) / name + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + for file_path in sorted(folder.rglob("*")): + if file_path.is_file(): + zipf.write(file_path, arcname=str(file_path.relative_to(folder))) + return str(zip_path) diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml new file mode 100644 index 000000000..67863ec25 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/first.toml @@ -0,0 +1,15 @@ +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml new file mode 100644 index 000000000..abee428a5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/entities/second.toml @@ -0,0 +1,17 @@ +# Redeclares the same entity key as first.toml, which would silently overwrite +# it if we didn't check. +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1 Again" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/duplicate_entities/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep b/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep new file mode 100644 index 000000000..1c3071fb4 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/empty_archive/.gitkeep @@ -0,0 +1 @@ +# Intentionally empty: an archive with no package.toml at all. diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml new file mode 100644 index 000000000..f284a5753 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_lp_key/package.toml @@ -0,0 +1,12 @@ +# The [learning_package] table is missing its mandatory "key" field. +[meta] +format_version = 1 +created_by = "lp_user" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml new file mode 100644 index 000000000..07d64b11c --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/missing_meta/package.toml @@ -0,0 +1,7 @@ +# The mandatory [meta] table is missing entirely. +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml new file mode 100644 index 000000000..993a94d82 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/entities/mystery.toml @@ -0,0 +1,16 @@ +# "chapter" is not a container type this version of the code can build. +[entity] +can_stand_alone = true +key = "chapter1-abc123" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.chapter] + +[[version]] +title = "Chapter 1" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unknown_container/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml new file mode 100644 index 000000000..4e53fd2f0 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/entities/unit1.toml @@ -0,0 +1,19 @@ +# This unit lists a child that isn't defined anywhere in the archive. +[entity] +can_stand_alone = true +key = "unit1-b7eafb" +created = 2025-09-04T22:51:59.271334Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.container.unit] + +[[version]] +title = "Unit 1" +version_num = 1 + +[version.container] +children = ["xblock.v1:html:does-not-exist"] diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml new file mode 100644 index 000000000..7fe7bf912 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unresolved_child/package.toml @@ -0,0 +1,13 @@ +[meta] +format_version = 1 +created_by = "lp_user" +created_by_email = "lp_user@example.com" +created_at = 2025-10-05T18:23:45.180535Z +origin_server = "cms.test" + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +description = "" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml b/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml new file mode 100644 index 000000000..4a2c10b43 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/fixtures/broken/unsupported_format_version/package.toml @@ -0,0 +1,11 @@ +# We only know how to read format_version 1.x. +[meta] +format_version = 2 +created_by = "lp_user" +created_at = 2025-10-05T18:23:45.180535Z + +[learning_package] +title = "Library test" +key = "lib:WGU:LIB_C001" +created = 2025-08-19T04:25:10.988166Z +updated = 2025-08-19T04:25:10.988166Z diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml new file mode 100644 index 000000000..9abcf1046 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/collections/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[collection] +title = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml new file mode 100644 index 000000000..5ff8122fe --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/broken_collection/package.toml @@ -0,0 +1,15 @@ +# This is a fully specified package TOML file with no errors, like we would +# expect from an Ulmo instance. +[meta] +format_version = 1 +created_by = "eddy" +created_by_email = "eddy@axim.org" +created_at = 2026-03-11T19:20:20.394360Z +origin_server = "studio.local.openedx.io" + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" +description = "My very fun library! 🐢" +created = 2026-02-11T16:32:47.524556Z +updated = 2026-02-20T16:32:47.524556Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml index e69de29bb..9abcf1046 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/broken.toml @@ -0,0 +1,3 @@ +# This is just malformed TOML. +[collection] +title = diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml index e69de29bb..09efb6975 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_1.toml @@ -0,0 +1,6 @@ +[collection] +title = "Practice Exams" +key = "dupe-collection-key" +description = "" +created = 2026-03-11T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml index e69de29bb..024fc2277 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/dupe_2.toml @@ -0,0 +1,8 @@ +# Same key as dupe_1.toml, which is not allowed -- Collection keys must be +# unique within a Learning Package. +[collection] +title = "Difficult Problems" +key = "dupe-collection-key" +description = "" +created = 2026-03-12T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml index e69de29bb..586a2439a 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/fields_not_in_table.toml @@ -0,0 +1,8 @@ +# The problem here is that title and key are not in a table, and we don't allow +# that. +title = "Difficult Problems" +key = "difficult-problems" + +[collection] +description = "" +created = 2026-03-11T19:20:20.394360Z diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml index e69de29bb..9d5a89d94 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/missing_collection_table.toml @@ -0,0 +1,7 @@ +# Typo: [collektion] instead of [collection] +[collektion] +title = "Difficult Problems" +key = "difficult-problems" +description = "" +created = 2026-03-11T19:20:20.394360Z +entities = [] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml new file mode 100644 index 000000000..c2ccd0582 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/collections/normal.toml @@ -0,0 +1,9 @@ +[collection] +title = "Difficult Problems" +key = "difficult-problems" +description = "The tricky ones. 🐢" +created = 2026-03-11T19:20:20.394360Z +entities = [ + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000001", + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000002", +] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_1.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml new file mode 100644 index 000000000..e332af3b5 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/duplicate_entities/entities/dupe_2.toml @@ -0,0 +1,4 @@ +[entity] +can_stand_alone = true +key = "dupe-key" +created = 2025-10-31T16:41:57.691331Z \ No newline at end of file diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep b/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep new file mode 100644 index 000000000..847e98cad --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/empty_archive/.gitkeep @@ -0,0 +1,2 @@ +# Intentionally empty: this directory exists so that fsspec has something +# to point at for the "archive with nothing in it" test case. diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml index e69de29bb..5448ff34d 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/missing_versions.toml @@ -0,0 +1,13 @@ +# An entity with no [[version]] tables at all. Extraction should still succeed +# and give us an empty "versions" list -- it's the validation step's job to +# decide whether that's acceptable. +[entity] +can_stand_alone = true +key = "no-versions-c0ffee" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] + +[entity.published] + +[entity.container.unit] diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml index e69de29bb..d86a9d4ad 100644 --- a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component.toml @@ -0,0 +1,20 @@ +[entity] +can_stand_alone = true +key = "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 3 + +[entity.published] +version_num = 2 + +# ### Versions + +[[version]] +title = "Text" +version_num = 3 + +[[version]] +title = "Text" +version_num = 2 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml new file mode 100644 index 000000000..e4fe6d5af --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v2/block.xml @@ -0,0 +1 @@ +

Version 2 text.

diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml new file mode 100644 index 000000000..50edb9d16 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/block.xml @@ -0,0 +1 @@ +

Version 3 text.

diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png new file mode 100644 index 000000000..9d17883bc --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/normal_component/component_versions/v3/static/figure.png @@ -0,0 +1 @@ +PNG-ish bytes for testing diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml new file mode 100644 index 000000000..ab7a14cac --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/entities/unknown_table.toml @@ -0,0 +1,26 @@ +# Neither [entity.future_thing] nor [future_top_level] is something this version +# of the code knows about. The nested one is kept, so that a newer export can add +# entity attributes without older code choking on them (and so we can warn about +# probable typos). The top-level one is outside the entity, so it isn't part of +# the entity data we assemble. +[entity] +can_stand_alone = true +key = "unknown-table-abc123" +created = 2026-04-08T15:22:12.780012Z + +[entity.draft] +version_num = 1 + +[entity.published] + +[entity.future_thing] +some_setting = "hello" + +[future_top_level] +irrelevant = true + +# ### Versions + +[[version]] +title = "Some Entity" +version_num = 1 diff --git a/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml new file mode 100644 index 000000000..d23b42927 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/payload_test_data/root_packages/unsupported_format_version_true.toml @@ -0,0 +1,8 @@ +# `true` is not a version number. This matters because Python's bool is a +# subclass of int, so a naive numeric check would read this as version 1. +[meta] +format_version = true + +[learning_package] +title = "Fun Library" +key = "lib:Axim:FunLib" diff --git a/tests/openedx_content/applets/backup_restore/test_archive.py b/tests/openedx_content/applets/backup_restore/test_archive.py new file mode 100644 index 000000000..89befdf6e --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_archive.py @@ -0,0 +1,69 @@ +""" +Tests for resolving an archive location into a filesystem we can read. + +These tests are strictly for the archive module, and therefore don't need Django +to run. +""" +import tempfile +import zipfile +from pathlib import Path +from unittest import TestCase + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem + +from openedx_content.applets.backup_restore import archive +from openedx_content.applets.backup_restore.errors import ArchiveNotReadableError + + +class ReadFsForPathTest(TestCase): + """Tests for resolving a path into a readable filesystem.""" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = Path(self._tmp.name) + + def _make_zip(self, name: str) -> Path: + zip_path = self.tmp_path / name + with zipfile.ZipFile(zip_path, "w") as zipf: + zipf.writestr("package.toml", "[meta]\nformat_version = 1\n") + return zip_path + + def test_directory(self): + contents_dir = self.tmp_path / "unzipped" + contents_dir.mkdir() + (contents_dir / "package.toml").write_text("[meta]\nformat_version = 1\n") + + fs = archive.read_fs_for_path(str(contents_dir)) + + assert isinstance(fs, DirFileSystem) + assert fs.exists("package.toml") + + def test_zip_file(self): + fs = archive.read_fs_for_path(str(self._make_zip("backup.zip"))) + + assert isinstance(fs, ZipFileSystem) + assert fs.exists("package.toml") + + def test_zip_file_uppercase_suffix(self): + """Suffix matching is case-insensitive.""" + fs = archive.read_fs_for_path(str(self._make_zip("BACKUP.ZIP"))) + + assert isinstance(fs, ZipFileSystem) + + def test_nonexistent_path(self): + missing = str(self.tmp_path / "not_here.zip") + + with self.assertRaises(ArchiveNotReadableError) as ctx: + archive.read_fs_for_path(missing) + assert ctx.exception.path == missing + + def test_file_that_is_not_a_zip(self): + not_a_zip = self.tmp_path / "package.toml" + not_a_zip.write_text("[meta]\nformat_version = 1\n") + + with self.assertRaises(ArchiveNotReadableError) as ctx: + archive.read_fs_for_path(str(not_a_zip)) + assert ctx.exception.path == str(not_a_zip) diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py new file mode 100644 index 000000000..1d841a68e --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -0,0 +1,579 @@ +""" +Tests for restoring a Learning Package from a backup archive. + +These drive the public API (``backup_restore.api``) rather than the loader +directly, so that the whole pipeline -- archive, payload, validation, loading -- +is exercised the way a caller would use it. + +The ``library_backup`` fixture is the workhorse here. It is a real archive +produced by the backup side, and several of its quirks are deliberate: + +* ``unit1`` has a blank version title, because untitled units are common in + content imported from courses. +* ``section1-extra-8ca126.toml`` and ``...-extra.toml`` have filenames that don't + match the entity key inside them, because the backup side hashes filenames to + avoid collisions. +* Its entities cover every draft/published combination we care about. +""" +import os +import tempfile +from datetime import datetime, timezone +from io import StringIO + +from django.contrib.auth import get_user_model +from django.core.management import CommandError, call_command +from django.test import TestCase + +from openedx_content.applets.backup_restore import api +from openedx_content.applets.backup_restore.errors import ( + ArchiveNotReadableError, + DuplicateFoundError, + MissingFileError, + RestoreFailedError, + SchemaError, + TableNotFoundError, + UnknownContainerTypeError, + UnresolvedChildError, + UnsupportedFormatError, +) +from openedx_content.applets.backup_restore.loading import Loader +from openedx_content.applets.backup_restore.results import generate_staged_package_ref +from openedx_content.applets.backup_restore.schema import CompletePackageInputData, EntityInputData +from openedx_content.applets.backup_restore.validation import ValidatedLearningPackageInput +from openedx_content.applets.collections import api as collections_api +from openedx_content.applets.components import api as components_api +from openedx_content.applets.containers import api as containers_api +from openedx_content.applets.publishing import api as publishing_api +from test_utils.zip_file_utils import folder_to_zip_path + +User = get_user_model() + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") +LIBRARY_BACKUP_DIR = os.path.join(FIXTURES_DIR, "library_backup") + + +def broken_fixture(name: str) -> str: + return os.path.join(FIXTURES_DIR, "broken", name) + + +class RestoreTestCase(TestCase): + """Base test case for restore tests.""" + + def setUp(self): + super().setUp() + self.fixtures_folder = LIBRARY_BACKUP_DIR + self.package_ref = "lib:WGU:LIB_C001" + self.user = User.objects.create_user(username="lp_user", password="12345") + + def as_zip(self, folder: str) -> str: + """Write a fixture folder out as a real zip file and return its path.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + return folder_to_zip_path(folder, tmp_dir.name) + + +class RestoreLearningPackageTest(RestoreTestCase): + """Restoring a well-formed archive.""" + + def test_restore_with_explicit_package_ref(self): + result = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + assert result.status == "success" + assert result.log_file_error is None + + restored = result.lp_restored_data + assert restored.package_ref == "lib-xx:WGU:LIB_C001" + assert restored.archive_package_ref == "lib:WGU:LIB_C001" + assert restored.archive_org_code == "WGU" + assert restored.archive_package_code == "LIB_C001" + assert restored.title == "Library test" + assert restored.num_containers == 3 + assert restored.num_sections == 1 + assert restored.num_subsections == 1 + assert restored.num_units == 1 + assert restored.num_components == 7 + assert restored.num_collections == 1 + + lp = publishing_api.LearningPackage.objects.filter( + package_ref="lib-xx:WGU:LIB_C001" + ).first() + assert lp is not None, "Learning package was not restored." + + def test_learning_package_fields_come_from_the_archive(self): + result = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + lp = publishing_api.LearningPackage.objects.get(id=result.lp_restored_data.id) + assert lp.title == "Library test" + assert lp.description == "" + assert lp.created == datetime(2025, 8, 19, 4, 25, 10, 988166, tzinfo=timezone.utc) + + def test_restore_with_staged_package_ref(self): + """Without an explicit ref we generate one namespaced to the user.""" + result = api.load_learning_package(self.fixtures_folder, user=self.user) + + assert result.status == "success" + restored_ref = result.lp_restored_data.package_ref + assert result.lp_restored_data.archive_package_ref == "lib:WGU:LIB_C001" + assert restored_ref.startswith("lp-restore:lp_user:WGU:LIB_C001:") + + lp = publishing_api.LearningPackage.objects.filter( + package_ref=restored_ref + ).first() + assert lp is not None, "Learning package with staged ref was not restored." + + def test_backup_metadata(self): + result = api.load_learning_package(self.fixtures_folder, user=self.user) + + assert result.status == "success" + assert result.backup_metadata.format_version == 1 + assert result.backup_metadata.created_by == "lp_user" + assert result.backup_metadata.created_by_email == "lp_user@example.com" + assert result.backup_metadata.created_at == datetime( + 2025, 10, 5, 18, 23, 45, 180535, tzinfo=timezone.utc + ) + assert result.backup_metadata.original_server == "cms.test" + + def test_restore_from_zip_matches_restore_from_directory(self): + """ + A zip archive and the same archive unzipped must load identically. + + Reading directly from a directory is new -- the old implementation only + accepted zip files -- so it's worth pinning that the two agree. + """ + from_dir = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib:from:dir" + ) + from_zip = api.load_learning_package( + self.as_zip(self.fixtures_folder), user=self.user, package_ref="lib:from:zip" + ) + + def comparable(result): + data = dict(vars(result.lp_restored_data)) + # These differ by construction. + del data["id"] + del data["package_ref"] + return data + + assert comparable(from_dir) == comparable(from_zip) + assert vars(from_dir.backup_metadata) == vars(from_zip.backup_metadata) + + def test_blank_container_title(self): + """ + Restoring should succeed when a container version has a blank title. + + Blank titles are legal and common -- content imported from courses + (e.g. via the modulestore migrator) frequently has untitled units, and + such content can be backed up. Restoring that same archive must work. + + The ``library_backup`` fixture's ``unit1`` deliberately has a blank + title to exercise this path. + """ + result = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + + assert result.status == "success" + lp = publishing_api.LearningPackage.objects.get(id=result.lp_restored_data.id) + unit = containers_api.get_containers(learning_package_id=lp.id).get( + publishable_entity__entity_ref="unit1-b7eafb" + ) + draft_version = publishing_api.get_draft_version(unit.publishable_entity.id) + assert draft_version.title == "" + + def test_entity_key_need_not_match_filename(self): + """ + The key inside the file wins, not the filename. + + The backup side hashes filenames to keep them unique and filesystem-safe, + so the two routinely differ. Two fixture files are named to make sure we + don't accidentally start trusting the filename. + """ + result = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + lp_id = result.lp_restored_data.id + + entity_refs = set( + publishing_api.get_publishable_entities(lp_id).values_list( + "entity_ref", flat=True + ) + ) + assert "section1-8ca126" in entity_refs + assert "section1-extra-8ca126" not in entity_refs + assert "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" in entity_refs + + +class RestoreContentTest(RestoreTestCase): + """Verifies what actually landed in the database.""" + + def setUp(self): + super().setUp() + result = api.load_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" + ) + self.lp = publishing_api.LearningPackage.objects.get( + id=result.lp_restored_data.id + ) + + def draft_and_published(self, entity_id): + return ( + publishing_api.get_draft_version(entity_id), + publishing_api.get_published_version(entity_id), + ) + + def test_containers(self): + """Verify the containers and their versions were restored correctly.""" + container_qs = containers_api.get_containers(learning_package_id=self.lp.id) + expected = { + # entity_ref: (type, draft version, published version) + "unit1-b7eafb": ("unit", 2, 2), + "subsection1-48afa3": ("subsection", 2, None), + "section1-8ca126": ("section", 2, None), + } + assert {c.entity_ref for c in container_qs} == set(expected) + + for container in container_qs: + container_type, draft_num, published_num = expected[container.entity_ref] + assert containers_api.get_container_type_code_of(container) == container_type + assert container.created_by is not None + assert container.created_by.username == "lp_user" + + draft, published = self.draft_and_published(container.publishable_entity.id) + assert draft is not None + assert draft.version_num == draft_num + assert draft.created_by.username == "lp_user" + if published_num is None: + assert published is None + else: + assert published is not None + assert published.version_num == published_num + assert published.created_by.username == "lp_user" + + def test_components(self): + """ + Verify the components and their versions were restored correctly. + + The version numbers here are the interesting part: they cover draft == + published, draft ahead of published, and never-published. + """ + expected = { + # entity_ref: (component type, draft version, published version) + "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27": + ("drag-and-drop-v2", 2, None), + "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37": ("html", 5, 4), + "xblock.v1:openassessment:1ee38208-a585-4455-a27e-4930aa541f53": + ("openassessment", 2, None), + "xblock.v1:problem:256739e8-c2df-4ced-bd10-8156f6cfa90b": ("problem", 2, None), + "xblock.v1:survey:6681da3f-b056-4c6e-a8f9-040967907471": ("survey", 1, None), + "xblock.v1:video:22601ebd-9da8-430b-9778-cfe059a98568": ("video", 3, None), + "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2": ("html", 2, 2), + } + component_qs = components_api.get_components(self.lp.id) + assert {c.entity_ref for c in component_qs} == set(expected) + + for component in component_qs: + type_name, draft_num, published_num = expected[component.entity_ref] + assert component.component_type.name == type_name + assert component.component_type.namespace == "xblock.v1" + assert component.created_by is not None + assert component.created_by.username == "lp_user" + + draft, published = self.draft_and_published(component.publishable_entity.id) + assert draft is not None + assert draft.version_num == draft_num + assert draft.created_by.username == "lp_user" + if published_num is None: + assert published is None + else: + assert published is not None + assert published.version_num == published_num + assert published.created_by.username == "lp_user" + + def test_block_xml_becomes_text_media(self): + component = components_api.get_components(self.lp.id).get( + publishable_entity__entity_ref=( + "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27" + ) + ) + draft = publishing_api.get_draft_version(component.publishable_entity.id) + + media = draft.componentversion.media.all() + assert media.count() == 1 + block_xml = media.first() + assert "

Version 2 text.

\n", + } + # ...while static assets are encoded as "fs:" pointers back into the + # archive, so that we don't hold binary files in memory. + v3_media = versions_by_num[3]["component"]["media"] + assert v3_media["block.xml"] == "

Version 3 text.

\n" + assert v3_media["static/figure.png"].startswith("fs:") + assert v3_media["static/figure.png"].endswith( + "normal_component/component_versions/v3/static/figure.png" + ) def test_normal_container(self): ref, data = payload.extract_entity_data(self.fs, "normal_container.toml") @@ -227,3 +283,151 @@ def test_normal_container(self): } +class ExtractCollectionDataTest(TestCase): + """Tests for reading a single collection TOML file.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.fs = DirFileSystem(TEST_DATA_ROOT / "collections") + + @classmethod + def tearDownClass(cls): + del cls.fs + super().tearDownClass() + + def test_broken_toml(self): + with self.assertRaises(payload.InvalidTOMLError) as ctx: + payload.extract_collection_data(self.fs, "broken.toml") + assert ctx.exception.path == "broken.toml" + + def test_fields_not_in_table(self): + with self.assertRaises(payload.FieldsNotInTable) as ctx: + payload.extract_collection_data(self.fs, "fields_not_in_table.toml") + assert ctx.exception.path == "fields_not_in_table.toml" + assert ctx.exception.fields == ["key", "title"] + + def test_missing_collection_table(self): + with self.assertRaises(payload.TableNotFoundError) as ctx: + payload.extract_collection_data(self.fs, "missing_collection_table.toml") + assert ctx.exception.path == "missing_collection_table.toml" + assert ctx.exception.table == "collection" + assert "[collection]" in str(ctx.exception) + + def test_normal(self): + data = payload.extract_collection_data(self.fs, "normal.toml") + assert data == { + "title": "Difficult Problems", + "key": "difficult-problems", + "description": "The tricky ones. 🐢", + "created": datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), + "entities": [ + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000001", + "xblock.v1:problem:e1f4b0a2-0000-4000-8000-000000000002", + ], + # Tracked so that validation errors can name the file they came from. + "src_path": "normal.toml", + } + + def test_dupes_are_not_caught_here(self): + """ + Duplicate Collection keys are a validation-level problem, not an + extraction-level one. + + Unlike entities -- which are assembled into a dict keyed by entity ref, + so a duplicate would silently overwrite -- collections are assembled into + a list. Nothing is lost at this layer, so we extract both and let + CompletePackageInputData.check_for_duplicate_keys reject them. + """ + dupe_1 = payload.extract_collection_data(self.fs, "dupe_1.toml") + dupe_2 = payload.extract_collection_data(self.fs, "dupe_2.toml") + assert dupe_1["key"] == dupe_2["key"] == "dupe-collection-key" + assert dupe_1["src_path"] != dupe_2["src_path"] + + +class ExtractUnvalidatedLearningPackageTest(TestCase): + """ + Tests for assembling a whole archive, rather than a single file. + + The important behavior here is that this function *collects* errors instead + of raising them, so that someone repairing an archive by hand sees every + problem at once. + """ + + def test_normal(self): + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + unvalidated = payload.extract_unvalidated_learning_package(fs) + + assert unvalidated.errors == [] + assert unvalidated.raw_data["learning_package"]["key"] == "lib:WGU:LIB_C001" + assert unvalidated.raw_data["meta"]["format_version"] == 1 + assert len(unvalidated.raw_data["entities"]) == 10 + assert len(unvalidated.raw_data["collections"]) == 1 + + def test_entity_path_mapping_uses_declared_key(self): + """ + The mapping is keyed by the entity's declared key, not its filename. + + Two fixture files deliberately have names that don't match the key + inside them, because the export side hashes filenames to avoid + collisions. + """ + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + unvalidated = payload.extract_unvalidated_learning_package(fs) + + assert ( + unvalidated.entity_path_mapping["section1-8ca126"] + == "entities/section1-extra-8ca126.toml" + ) + assert unvalidated.entity_path_mapping[ + "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" + ].endswith("c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2-extra.toml") + + def test_missing_root_package_is_collected_not_raised(self): + fs = DirFileSystem(TEST_DATA_ROOT / "empty_archive") + unvalidated = payload.extract_unvalidated_learning_package(fs) + + assert len(unvalidated.errors) == 1 + error = unvalidated.errors[0] + assert isinstance(error, payload.MissingFileError) + assert error.path == "package.toml" + + # We still return a usable object, just an empty one. + assert unvalidated.raw_data["entities"] == {} + assert unvalidated.raw_data["collections"] == [] + + def test_duplicate_entities_are_collected(self): + fs = DirFileSystem(TEST_DATA_ROOT / "duplicate_entities") + unvalidated = payload.extract_unvalidated_learning_package(fs) + + duplicate_errors = [ + err for err in unvalidated.errors + if isinstance(err, payload.DuplicateFoundError) + ] + assert len(duplicate_errors) == 1 + # The first file to declare the key wins; the second is the error. + assert duplicate_errors[0].original_path == "entities/dupe_1.toml" + assert duplicate_errors[0].path == "entities/dupe_2.toml" + assert "dupe-key" in unvalidated.raw_data["entities"] + + def test_static_assets_are_not_mistaken_for_entities(self): + """ + TOML files under component_versions/ are static assets, not entities. + """ + fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + paths = payload.get_entity_file_paths(fs) + + assert paths == sorted(paths) # deterministic ordering + assert all("/component_versions/" not in path for path in paths) + + def test_collection_errors_are_collected(self): + fs = DirFileSystem(TEST_DATA_ROOT / "broken_collection") + unvalidated = payload.extract_unvalidated_learning_package(fs) + + assert len(unvalidated.errors) == 1 + error = unvalidated.errors[0] + assert isinstance(error, payload.InvalidTOMLError) + assert error.path == "collections/broken.toml" + + # The rest of the archive still came through. + assert unvalidated.raw_data["learning_package"]["key"] == "lib:Axim:FunLib" diff --git a/tests/openedx_content/applets/backup_restore/test_restore.py b/tests/openedx_content/applets/backup_restore/test_restore.py deleted file mode 100644 index 693e886cf..000000000 --- a/tests/openedx_content/applets/backup_restore/test_restore.py +++ /dev/null @@ -1,414 +0,0 @@ -"""Tests for the lp_load management command.""" -import os -from datetime import datetime, timezone -from io import StringIO -from unittest.mock import patch - -from django.contrib.auth import get_user_model -from django.core.management import call_command -from django.test import TestCase - -from openedx_content.applets.backup_restore.zipper import LearningPackageUnzipper, generate_staged_package_ref -from openedx_content.applets.collections import api as collections_api -from openedx_content.applets.components import api as components_api -from openedx_content.applets.containers import api as containers_api -from openedx_content.applets.publishing import api as publishing_api -from test_utils.zip_file_utils import folder_to_inmemory_zip - -User = get_user_model() - - -class RestoreTestCase(TestCase): - """Base test case for restore tests.""" - - def setUp(self): - super().setUp() - self.fixtures_folder = os.path.join(os.path.dirname(__file__), "fixtures/library_backup") - self.zip_file = folder_to_inmemory_zip(self.fixtures_folder) - self.package_ref = "lib:WGU:LIB_C001" - self.user = User.objects.create_user(username='lp_user', password='12345') - - -class RestoreLearningPackageCommandTest(RestoreTestCase): - """Tests for the lp_load management command.""" - - @patch("openedx_content.applets.backup_restore.api.load_learning_package") - def test_restore_command(self, mock_load_learning_package): - # Mock load_learning_package to return our in-memory zip file - restore_result = LearningPackageUnzipper(self.zip_file, user=self.user).load() - mock_load_learning_package.return_value = restore_result - - out = StringIO() - # You can pass any dummy path, since load_learning_package is mocked - call_command("lp_load", "dummy.zip", "lp_user", stdout=out) - - lp = self.verify_lp(restore_result["lp_restored_data"]["package_ref"]) - self.verify_containers(lp) - self.verify_components(lp) - self.verify_collections(lp) - - def verify_lp(self, package_ref): - """Verify the learning package was restored correctly.""" - lp = publishing_api.LearningPackage.objects.filter(package_ref=package_ref).first() - assert lp is not None, "Learning package was not restored." - assert lp.title == "Library test" - assert lp.description == "" - return lp - - def verify_containers(self, lp): - """Verify the containers and their versions were restored correctly.""" - container_qs = containers_api.get_containers(learning_package_id=lp.id) - expected_container_keys = ["unit1-b7eafb", "subsection1-48afa3", "section1-8ca126"] - - for container in container_qs: - assert container.entity_ref in expected_container_keys - draft_version = publishing_api.get_draft_version(container.publishable_entity.id) - published_version = publishing_api.get_published_version(container.publishable_entity.id) - assert container.created_by is not None - assert container.created_by.username == "lp_user" - # The unit has been published. The other two containers haven't been. - # It's important that we test with at least one published container in order to - # fully cover _create_container in zipper.py. - if container.entity_ref == "unit1-b7eafb": - assert containers_api.get_container_type_code_of(container) == "unit" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is not None - assert published_version.version_num == 2 - assert published_version.created_by is not None - assert published_version.created_by.username == "lp_user" - elif container.entity_ref == "subsection1-48afa3": - assert containers_api.get_container_type_code_of(container) == "subsection" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - elif container.entity_ref == "section1-8ca126": - assert containers_api.get_container_type_code_of(container) == "section" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - else: - assert False, f"Unexpected container key: {container.entity_ref}" - - def verify_components(self, lp): - # pylint: disable=too-many-statements - """Verify the components and their versions were restored correctly.""" - component_qs = components_api.get_components(lp.id) - expected_component_keys = [ - "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27", - "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37", - "xblock.v1:openassessment:1ee38208-a585-4455-a27e-4930aa541f53", - "xblock.v1:problem:256739e8-c2df-4ced-bd10-8156f6cfa90b", - "xblock.v1:survey:6681da3f-b056-4c6e-a8f9-040967907471", - "xblock.v1:video:22601ebd-9da8-430b-9778-cfe059a98568", - "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" - ] - for component in component_qs: - assert component.entity_ref in expected_component_keys - draft_version = publishing_api.get_draft_version(component.publishable_entity.id) - published_version = publishing_api.get_published_version(component.publishable_entity.id) - assert component.created_by is not None - assert component.created_by.username == "lp_user" - if component.entity_ref == "xblock.v1:drag-and-drop-v2:4d1b2fac-8b30-42fb-872d-6b10ab580b27": - assert component.component_type.name == "drag-and-drop-v2" - assert component.component_type.namespace == "xblock.v1" - assert draft_version is not None - assert draft_version.version_num == 2 - assert draft_version.created_by is not None - assert draft_version.created_by.username == "lp_user" - assert published_version is None - # Get the content associated with this component - all_media = draft_version.componentversion.media.all() - media = all_media.first() if all_media.exists() else None - assert media is not None - assert " dict: + """The smallest input we consider loadable.""" + return { + "meta": {"format_version": 1}, + "learning_package": {"key": "lib:Axim:FunLib"}, + "entities": {}, + "collections": [], + **overrides, + } + + +class ContainerDiscriminationTest(TestCase): + """ + The container union has to resolve to the right model. + + All three container models allow extra fields, so if the discriminating field + were optional every one of them would validate every dict, and the union + would always collapse to whichever model happens to be listed first. That + failure is silent -- everything would load as a Unit -- so it's worth pinning + down explicitly. + """ + + def _container_for(self, raw): + entity = EntityInputData.model_validate({"created": CREATED, "container": raw}) + return entity.container + + def test_section(self): + assert isinstance(self._container_for({"section": {}}), SectionInputData) + + def test_subsection(self): + assert isinstance(self._container_for({"subsection": {}}), SubsectionInputData) + + def test_unit(self): + assert isinstance(self._container_for({"unit": {}}), UnitInputData) + + def test_unknown_container_type_falls_through_to_dict(self): + """ + A container type we don't recognize is still captured. + + We keep the raw dict rather than erroring here so that validation can + report a useful message. Loading rejects it. + """ + container = self._container_for({"chapter": {}}) + assert isinstance(container, dict) + assert container == {"chapter": {}} + + def test_no_container_means_component(self): + assert self._container_for(None) is None + + def test_absent_container_defaults_to_none(self): + entity = EntityInputData.model_validate({"created": CREATED}) + assert entity.container is None + + +class StringConstraintsTest(TestCase): + """ + The ref/code constraints have to actually be applied. + + These were declared with a trailing comma at one point, which made them + tuples rather than StringConstraints, and pydantic silently ignored them. + """ + + def test_collection_key_is_stripped(self): + collection = CollectionInput.model_validate( + {"title": "Difficult Problems", "key": " difficult-problems "} + ) + assert collection.key == "difficult-problems" + + def test_collection_key_rejects_spaces(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "has spaces"}) + + def test_collection_key_rejects_slashes(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "has/slash"}) + + def test_collection_key_rejects_trailing_newline(self): + with self.assertRaises(ValidationError): + CollectionInput.model_validate({"title": "T", "key": "trailing\nnewline"}) + + def test_learning_package_key_is_stripped(self): + lp = LearningPackageInputData.model_validate({"key": " lib:Axim:FunLib "}) + assert lp.key == "lib:Axim:FunLib" + + +class MetaInputDataTest(TestCase): + """Tests for the archive provenance metadata.""" + + def test_only_format_version_is_required(self): + """ + Everything else in [meta] is provenance information we can't trust + anyway, so an archive that omits it is still loadable. + """ + meta = MetaInputData.model_validate({"format_version": 1}) + + assert meta.format_version == 1 + assert meta.created_by is None + assert meta.created_by_email is None + assert meta.created_at is None + assert meta.origin_server is None + + def test_format_version_is_required(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate({}) + + def test_format_version_must_be_1(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate({"format_version": 2}) + + def test_email_is_validated(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate( + {"format_version": 1, "created_by_email": "not-an-email"} + ) + + +class LearningPackageInputDataTest(TestCase): + """Tests for the top-level Learning Package fields.""" + + def test_key_is_required(self): + with self.assertRaises(ValidationError) as ctx: + LearningPackageInputData.model_validate({"title": "Fun Library"}) + assert [err["loc"] for err in ctx.exception.errors()] == [("key",)] + + def test_title_has_a_default(self): + lp = LearningPackageInputData.model_validate({"key": "lib:Axim:FunLib"}) + assert lp.title == "Untitled Library" + + def test_blank_title_is_rejected(self): + with self.assertRaises(ValidationError): + LearningPackageInputData.model_validate({"key": "lib:A:B", "title": ""}) + + def test_dates_are_optional(self): + lp = LearningPackageInputData.model_validate({"key": "lib:Axim:FunLib"}) + assert lp.created is None + assert lp.updated is None + + def test_naive_datetimes_are_rejected(self): + """We store everything in UTC, so an ambiguous timestamp is an error.""" + with self.assertRaises(ValidationError): + LearningPackageInputData.model_validate( + {"key": "lib:A:B", "created": datetime(2026, 4, 8, 15, 22, 12)} + ) + + +class VersionInputTest(TestCase): + """Tests for entity versions and their draft/published pointers.""" + + def _entity_with_version(self, **version_overrides): + version = {"version_num": 1, "title": "Some Title", **version_overrides} + return EntityInputData.model_validate( + {"created": CREATED, "versions": [version]} + ) + + def test_blank_title_is_allowed(self): + """ + Blank titles are legal and common -- content imported from courses (e.g. + via the modulestore migrator) frequently has untitled units, and such + content can be backed up. Restoring that same archive must work. + """ + entity = self._entity_with_version(title="") + assert entity.versions[0].title == "" + + def test_version_num_must_be_positive(self): + with self.assertRaises(ValidationError): + self._entity_with_version(version_num=0) + + def test_draft_and_published_default_to_none(self): + entity = EntityInputData.model_validate({"created": CREATED}) + assert entity.draft.version_num is None + assert entity.published.version_num is None + + def test_empty_published_table_means_unpublished(self): + """ + An entity that has never been published exports an empty + [entity.published] table rather than omitting it. + """ + entity = EntityInputData.model_validate( + {"created": CREATED, "published": {}, "draft": {"version_num": 2}} + ) + assert entity.published.version_num is None + assert entity.draft.version_num == 2 + + +class CompletePackageInputDataTest(TestCase): + """Tests for validating a whole package document at once.""" + + def test_minimal_package(self): + data = CompletePackageInputData.model_validate(minimal_package()) + + assert data.learning_package.key == "lib:Axim:FunLib" + assert data.entities == {} + assert data.collections == [] + + def test_duplicate_collection_keys_are_rejected(self): + raw = minimal_package( + collections=[ + {"title": "One", "key": "same-key", "src_path": "collections/a.toml"}, + {"title": "Two", "key": "same-key", "src_path": "collections/b.toml"}, + ] + ) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + message = str(ctx.exception) + assert "same-key" in message + # The message should name both files, so it's actionable. + assert "collections/a.toml" in message + assert "collections/b.toml" in message + + def test_distinct_collection_keys_are_fine(self): + raw = minimal_package( + collections=[ + {"title": "One", "key": "key-one"}, + {"title": "Two", "key": "key-two"}, + ] + ) + data = CompletePackageInputData.model_validate(raw) + assert [c.key for c in data.collections] == ["key-one", "key-two"] + + def test_collection_entities_default_to_empty(self): + raw = minimal_package(collections=[{"title": "One", "key": "key-one"}]) + data = CompletePackageInputData.model_validate(raw) + assert data.collections[0].entities == [] + + def test_missing_meta_reports_against_meta(self): + raw = minimal_package() + del raw["meta"] + + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + assert ("meta",) in [err["loc"] for err in ctx.exception.errors()] + + def test_errors_are_reported_together(self): + """ + Pydantic collects every problem, which is what lets us show someone + repairing an archive all of their mistakes at once. + """ + raw = {"meta": {}, "learning_package": {}, "entities": {}, "collections": []} + + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + locations = {err["loc"] for err in ctx.exception.errors()} + assert ("meta", "format_version") in locations + assert ("learning_package", "key") in locations + + +class ForwardsCompatibilityTest(TestCase): + """ + Unrecognized fields are kept rather than dropped. + + Older installs need to load newer archives. We keep the unknown values rather + than ignoring them so that we can eventually warn about fields that look like + typos of ones we do know. + """ + + def test_unknown_fields_are_retained(self): + data = CompletePackageInputData.model_validate( + minimal_package( + learning_package={"key": "lib:A:B", "some_future_field": "hello"} + ) + ) + assert data.learning_package.some_future_field == "hello" + + def test_models_are_frozen(self): + data = CompletePackageInputData.model_validate(minimal_package()) + with self.assertRaises(ValidationError): + data.learning_package.title = "Changed" + + +class BlankMetadataTest(TestCase): + """ + Blank [meta] values must not block a restore. + + The backup side writes ``created_by_email`` unconditionally, and Django's + ``User.email`` defaults to an empty string, so archives with blank + provenance fields are routinely produced by our own export. + """ + + def test_blank_email(self): + meta = MetaInputData.model_validate( + {"format_version": 1, "created_by_email": ""} + ) + assert meta.created_by_email is None + + def test_blank_created_by(self): + meta = MetaInputData.model_validate({"format_version": 1, "created_by": " "}) + assert meta.created_by is None + + def test_blank_origin_server(self): + meta = MetaInputData.model_validate({"format_version": 1, "origin_server": ""}) + assert meta.origin_server is None + + def test_real_values_still_come_through(self): + meta = MetaInputData.model_validate({ + "format_version": 1, + "created_by": "eddy", + "created_by_email": "eddy@axim.org", + "origin_server": "studio.local.openedx.io", + }) + assert meta.created_by == "eddy" + assert meta.created_by_email == "eddy@axim.org" + assert meta.origin_server == "studio.local.openedx.io" + + def test_a_genuinely_bad_email_is_still_rejected(self): + with self.assertRaises(ValidationError): + MetaInputData.model_validate( + {"format_version": 1, "created_by_email": "not-an-email"} + ) diff --git a/tests/openedx_content/applets/backup_restore/test_validation.py b/tests/openedx_content/applets/backup_restore/test_validation.py new file mode 100644 index 000000000..57586ff03 --- /dev/null +++ b/tests/openedx_content/applets/backup_restore/test_validation.py @@ -0,0 +1,355 @@ +""" +Tests for turning extracted archive data into a validated model. + +The contract this module has to hold up: + +* ``validate`` never raises. Everything it finds goes onto ``.errors``, so that + someone repairing an archive by hand sees every problem in one pass. +* Errors name the archive file they came from. Pydantic reports against the + combined document we assemble internally, which nobody editing an archive has + ever seen, so we translate those locations back into file paths. +* Every cross-reference that ``loading.py`` relies on is checked here, so that a + broken archive produces a readable error instead of a traceback from the + middle of a database write. + +These tests are strictly for the validation module, and therefore don't need +Django to run. +""" +from datetime import datetime, timezone +from pathlib import Path +from unittest import TestCase + +from fsspec.implementations.dirfs import DirFileSystem + +from openedx_content.applets.backup_restore import validation +from openedx_content.applets.backup_restore.errors import ( + DuplicateVersionError, + InvalidTOMLError, + MalformedRefError, + MissingFileError, + MissingVersionError, + RestoreFailedError, + SchemaError, + UnknownContainerTypeError, + UnresolvedChildError, +) +from openedx_content.applets.backup_restore.payload import UnvalidatedLearningPackageInput + +FIXTURES_ROOT = Path(__file__).parent / "fixtures" +CREATED = datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc) + + +def component(**overrides) -> dict: + """Raw data for a Component entity (i.e. one with no container).""" + return {"can_stand_alone": True, "created": CREATED, "versions": [], **overrides} + + +def container(kind: str, **overrides) -> dict: + """Raw data for a Container entity of the given kind.""" + return component(container={kind: {}}, **overrides) + + +def version(version_num: int, children=None, **overrides) -> dict: + raw = {"version_num": version_num, "title": f"v{version_num}", **overrides} + if children is not None: + raw["container"] = {"children": children} + return raw + + +def unvalidated(entities=None, collections=None, errors=None, path_mapping=None, **overrides): + """Build an UnvalidatedLearningPackageInput without going through files.""" + raw_data = { + "meta": {"format_version": 1}, + "learning_package": {"key": "lib:Axim:FunLib"}, + "entities": entities if entities is not None else {}, + "collections": collections if collections is not None else [], + **overrides, + } + return UnvalidatedLearningPackageInput( + raw_data=raw_data, + errors=errors or [], + fs=DirFileSystem(FIXTURES_ROOT), + entity_path_mapping=path_mapping or {}, + ) + + +class ValidateSuccessTest(TestCase): + """Archives that validate cleanly.""" + + def test_minimal_package(self): + result = validation.validate(unvalidated()) + + assert result.errors == [] + assert result.data is not None + assert result.data.learning_package.key == "lib:Axim:FunLib" + + def test_fs_is_passed_through(self): + """The loader needs the filesystem later, to read static assets.""" + source = unvalidated() + result = validation.validate(source) + assert result.fs is source.fs + + def test_well_formed_container_tree(self): + result = validation.validate(unvalidated(entities={ + "section-1": container("section", versions=[version(1, ["subsection-1"])]), + "subsection-1": container("subsection", versions=[version(1, ["unit-1"])]), + "unit-1": container("unit", versions=[version(1, ["xblock.v1:html:abc"])]), + "xblock.v1:html:abc": component(versions=[version(1)]), + })) + + assert result.errors == [] + + +class ExtractionErrorsCarryThroughTest(TestCase): + """ + Extraction errors are already BackupRestoreErrors, so they come through as + themselves -- not flattened into some other error record. + """ + + def test_error_type_and_path_are_preserved(self): + extraction_error = InvalidTOMLError( + "Entity", details="bad token", path="entities/broken.toml" + ) + result = validation.validate(unvalidated(errors=[extraction_error])) + + assert result.errors[0] is extraction_error + assert isinstance(result.errors[0], InvalidTOMLError) + assert result.errors[0].path == "entities/broken.toml" + + def test_extraction_errors_are_reported_alongside_schema_errors(self): + result = validation.validate( + unvalidated( + errors=[MissingFileError("Root Package", path="package.toml")], + learning_package={}, # also missing its key + ) + ) + + error_types = {type(err) for err in result.errors} + assert MissingFileError in error_types + assert SchemaError in error_types + + +class SchemaErrorSourceMappingTest(TestCase): + """ + Pydantic's `loc` gets translated back into "which file is wrong". + """ + + def test_learning_package_errors_point_at_package_toml(self): + result = validation.validate(unvalidated(learning_package={})) + + assert result.data is None + error = result.errors[0] + assert isinstance(error, SchemaError) + assert error.path == "package.toml" + assert error.location == ("learning_package", "key") + + def test_meta_errors_point_at_package_toml(self): + result = validation.validate(unvalidated(meta={})) + + error = result.errors[0] + assert error.path == "package.toml" + assert error.location == ("meta", "format_version") + + def test_entity_errors_point_at_the_entity_file(self): + result = validation.validate(unvalidated( + entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, + path_mapping={"unit-1": "entities/unit1-b7eafb.toml"}, + )) + + error = result.errors[0] + assert error.path == "entities/unit1-b7eafb.toml" + # The entity ref is dropped, since the file only describes one entity. + assert error.location == ("versions", 0, "title") + + def test_entity_errors_fall_back_when_path_is_unknown(self): + result = validation.validate(unvalidated( + entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, + )) + + assert result.errors[0].path == "entities/unit-1" + + def test_collection_errors_point_at_the_collection_file(self): + result = validation.validate(unvalidated( + collections=[{"key": "no-title", "src_path": "collections/broken.toml"}], + )) + + error = result.errors[0] + assert error.path == "collections/broken.toml" + assert error.location == ("title",) + + def test_error_message_includes_location(self): + result = validation.validate(unvalidated( + entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, + path_mapping={"unit-1": "entities/unit1.toml"}, + )) + + assert "entities/unit1.toml: versions.0.title" in str(result.errors[0]) + + +class ConsistencyCheckTest(TestCase): + """ + Cross-references pydantic can't express. Each of these would otherwise be an + uncaught exception in the middle of loading. + """ + + def test_unresolved_child(self): + result = validation.validate(unvalidated( + entities={"unit-1": container("unit", versions=[version(1, ["nope"])])}, + path_mapping={"unit-1": "entities/unit1.toml"}, + )) + + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, UnresolvedChildError) + assert error.path == "entities/unit1.toml" + assert "nope" in error.message + + def test_draft_pointing_at_a_missing_version(self): + result = validation.validate(unvalidated(entities={ + "unit-1": container("unit", draft={"version_num": 7}, versions=[version(1)]), + })) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MissingVersionError) + assert "[entity.draft]" in result.errors[0].message + + def test_published_pointing_at_a_missing_version(self): + result = validation.validate(unvalidated(entities={ + "unit-1": container( + "unit", published={"version_num": 7}, versions=[version(1)] + ), + })) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MissingVersionError) + assert "[entity.published]" in result.errors[0].message + + def test_draft_and_published_may_be_absent(self): + """An entity that was created and then reset to published has neither.""" + result = validation.validate(unvalidated(entities={ + "unit-1": container("unit", versions=[version(1)]), + })) + assert result.errors == [] + + def test_duplicate_version_num(self): + result = validation.validate(unvalidated(entities={ + "unit-1": container("unit", versions=[version(2), version(2)]), + })) + + duplicate_errors = [ + err for err in result.errors if isinstance(err, DuplicateVersionError) + ] + assert len(duplicate_errors) == 1 + assert "version 2" in duplicate_errors[0].message + + def test_malformed_component_ref(self): + """ + Component refs are "{namespace}:{type}:{code}" -- the loader splits on + the colons to work out what kind of block to build. + """ + result = validation.validate(unvalidated(entities={ + "not-a-component-ref": component(versions=[version(1)]), + })) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], MalformedRefError) + + def test_container_refs_are_not_required_to_have_colons(self): + """Only Components derive meaning from the shape of their ref.""" + result = validation.validate(unvalidated(entities={ + "unit1-b7eafb": container("unit", versions=[version(1)]), + })) + assert result.errors == [] + + def test_unknown_container_type(self): + result = validation.validate(unvalidated( + entities={"thing-1": component(container={"chapter": {}})}, + path_mapping={"thing-1": "entities/thing1.toml"}, + )) + + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, UnknownContainerTypeError) + assert error.path == "entities/thing1.toml" + assert "chapter" in error.message + + def test_consistency_checks_are_skipped_when_the_schema_is_broken(self): + """ + If we couldn't build a model, there's nothing to cross-reference. We + report the schema errors rather than inventing consistency errors on top + of data we know is malformed. + """ + result = validation.validate(unvalidated( + meta={}, + entities={"unit-1": container("unit", versions=[version(1, ["nope"])])}, + )) + + assert result.data is None + assert all(isinstance(err, SchemaError) for err in result.errors) + + def test_all_problems_are_reported_together(self): + result = validation.validate(unvalidated(entities={ + "unit-1": container( + "unit", draft={"version_num": 9}, versions=[version(1, ["nope"])] + ), + "bad-component-ref": component(versions=[version(1)]), + })) + + error_types = {type(err) for err in result.errors} + assert error_types == { + UnresolvedChildError, + MissingVersionError, + MalformedRefError, + } + + +class RestoreFailedErrorTest(TestCase): + """Tests for how a batch of errors is reported.""" + + def test_as_text_matches_the_legacy_log_format(self): + error = RestoreFailedError([ + MissingFileError("Root Package", path="package.toml"), + InvalidTOMLError("Entity", details="bad token", path="entities/x.toml"), + ]) + + assert error.as_text() == ( + "Errors encountered during restore:\n" + "package.toml: Root Package file not found at expected path\n" + "entities/x.toml: Cannot decode TOML for Entity: bad token\n" + ) + + def test_errors_are_kept_for_inspection(self): + original = MissingFileError("Root Package", path="package.toml") + error = RestoreFailedError([original]) + + assert error.errors == [original] + + +class SourceMappingFallbackTest(TestCase): + """ + Errors we can't attribute to a file still get reported. + + These paths shouldn't come up in practice, but silently dropping an error + because we couldn't work out where it came from would be much worse than + reporting it without a filename. + """ + + def test_unrecognized_location_has_no_path(self): + result = validation.validate(unvalidated(entities="not-a-dict")) + + assert result.data is None + error = result.errors[0] + assert error.path is None + assert error.location == ("entities",) + assert str(error).startswith("None: entities:") + + def test_collection_without_a_src_path(self): + result = validation.validate(unvalidated( + collections=[{"key": "no-title"}], # no src_path to attribute it to + )) + + assert result.errors[0].path is None + + def test_schema_error_without_a_location(self): + error = SchemaError("something went wrong", path="package.toml") + assert str(error) == "package.toml: something went wrong" diff --git a/tox.ini b/tox.ini index ad8736086..2f3bb87bb 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,10 @@ match-dir = (?!migrations) [pytest] DJANGO_SETTINGS_MODULE = test_settings +; So that a bare `pytest` run (outside tox, which installs the package) can +; still find the source tree. +pythonpath = src +testpaths = tests addopts = --cov src --cov tests --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements site-packages filterwarnings = From 47ad45d84a6c4581d42aa9d670375b9c23010a41 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 15:19:21 -0400 Subject: [PATCH 05/14] temp: minor refactoring to maintain the old API function signature for backwards compatibility --- .../applets/backup_restore/api.py | 28 ++++++++--------- .../management/commands/lp_load.py | 4 +-- .../applets/backup_restore/test_loading.py | 30 +++++++++---------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index e2d4b32d2..9def63331 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -19,12 +19,12 @@ __all__ = [ "create_zip_file", + "create_learning_package", "load_learning_package", - "load_learning_package_as_dict", ] -def load_learning_package( +def create_learning_package( path_str: str, user: UserType, package_ref: str | None = None, @@ -45,14 +45,14 @@ def load_learning_package( loading.py: ValidatedLearningPackageInput → LearningPackage If ``package_ref`` is not supplied, we generate a staged one namespaced to - ``user``. We can't just use the ref from the archive, because the archive can - claim any ref it likes and the user may not be allowed to create it. + ``user``. We can't just use the ref from the archive, because the archive + can claim any ref it likes and the user may not be allowed to create it. Errors that can be raised: * ``ArchiveNotReadableError`` if we can't open ``path_str`` at all. - * ``RestoreFailedError`` if the archive's contents don't validate. Nothing is - written to the database in that case. + * ``RestoreFailedError`` if the archive's contents don't validate. Nothing + is written to the database in that case. Both descend from ``BackupRestoreError``. """ @@ -67,17 +67,17 @@ def load_learning_package( raise RestoreFailedError(validated_input.errors) loader = loading.Loader(validated_input) - archive_lp = loader.data.learning_package + archive_lp_input = loader.data.learning_package # LearningPackageInputData if package_ref is None: - package_ref = generate_staged_package_ref(archive_lp.key, user) + package_ref = generate_staged_package_ref(archive_lp_input.key, user) now = datetime.now(tz=timezone.utc) with atomic(savepoint=False): learning_package = publishing_api.create_learning_package( package_ref, - archive_lp.title, - description=archive_lp.description or "", - created=archive_lp.created or now, + archive_lp_input.title, + description=archive_lp_input.description or "", + created=archive_lp_input.created or now, ) load_target = loading.Loader.Target(learning_package, user, now) result = loader.load_into(load_target) @@ -85,13 +85,13 @@ def load_learning_package( return result -def load_learning_package_as_dict( +def load_learning_package( path_str: str, user: UserType, package_ref: str | None = None, ) -> dict: """ - ``load_learning_package``, in the dict shape the frontend currently expects. + ``create_learning_package``, in the dict shape that 1.0 clients expect. Returns a dict with the status of the operation and any errors encountered during that process, rather than raising. @@ -101,7 +101,7 @@ def load_learning_package_as_dict( ``BackupRestoreError``, so that this can eventually go away. """ try: - result = load_learning_package(path_str, user, package_ref) + result = create_learning_package(path_str, user, package_ref) except RestoreFailedError as err: return asdict( RestoreResult(status="error", log_file_error=StringIO(err.as_text())) diff --git a/src/openedx_content/management/commands/lp_load.py b/src/openedx_content/management/commands/lp_load.py index 3ff64e269..2bcd47e86 100644 --- a/src/openedx_content/management/commands/lp_load.py +++ b/src/openedx_content/management/commands/lp_load.py @@ -8,7 +8,7 @@ from django.core.management import CommandError from django.core.management.base import BaseCommand -from openedx_content.applets.backup_restore.api import load_learning_package +from openedx_content.applets.backup_restore.api import create_learning_package from openedx_content.applets.backup_restore.errors import BackupRestoreError, RestoreFailedError logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def handle(self, *args, **options): start_time = time.time() try: - result = load_learning_package(path, user=user, package_ref=package_ref) + result = create_learning_package(path, user=user, package_ref=package_ref) except RestoreFailedError as exc: # The archive is bad. Show every problem we found, not just the first. raise CommandError(exc.as_text()) from exc diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py index 1d841a68e..db79b784a 100644 --- a/tests/openedx_content/applets/backup_restore/test_loading.py +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -76,7 +76,7 @@ class RestoreLearningPackageTest(RestoreTestCase): """Restoring a well-formed archive.""" def test_restore_with_explicit_package_ref(self): - result = api.load_learning_package( + result = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -102,7 +102,7 @@ def test_restore_with_explicit_package_ref(self): assert lp is not None, "Learning package was not restored." def test_learning_package_fields_come_from_the_archive(self): - result = api.load_learning_package( + result = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -113,7 +113,7 @@ def test_learning_package_fields_come_from_the_archive(self): def test_restore_with_staged_package_ref(self): """Without an explicit ref we generate one namespaced to the user.""" - result = api.load_learning_package(self.fixtures_folder, user=self.user) + result = api.create_learning_package(self.fixtures_folder, user=self.user) assert result.status == "success" restored_ref = result.lp_restored_data.package_ref @@ -126,7 +126,7 @@ def test_restore_with_staged_package_ref(self): assert lp is not None, "Learning package with staged ref was not restored." def test_backup_metadata(self): - result = api.load_learning_package(self.fixtures_folder, user=self.user) + result = api.create_learning_package(self.fixtures_folder, user=self.user) assert result.status == "success" assert result.backup_metadata.format_version == 1 @@ -144,10 +144,10 @@ def test_restore_from_zip_matches_restore_from_directory(self): Reading directly from a directory is new -- the old implementation only accepted zip files -- so it's worth pinning that the two agree. """ - from_dir = api.load_learning_package( + from_dir = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib:from:dir" ) - from_zip = api.load_learning_package( + from_zip = api.create_learning_package( self.as_zip(self.fixtures_folder), user=self.user, package_ref="lib:from:zip" ) @@ -172,7 +172,7 @@ def test_blank_container_title(self): The ``library_backup`` fixture's ``unit1`` deliberately has a blank title to exercise this path. """ - result = api.load_learning_package( + result = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -192,7 +192,7 @@ def test_entity_key_need_not_match_filename(self): so the two routinely differ. Two fixture files are named to make sure we don't accidentally start trusting the filename. """ - result = api.load_learning_package( + result = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) lp_id = result.lp_restored_data.id @@ -212,7 +212,7 @@ class RestoreContentTest(RestoreTestCase): def setUp(self): super().setUp() - result = api.load_learning_package( + result = api.create_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) self.lp = publishing_api.LearningPackage.objects.get( @@ -367,7 +367,7 @@ def assert_refuses(self, fixture_name, error_type): packages_before = publishing_api.LearningPackage.objects.count() with self.assertRaises(RestoreFailedError) as ctx: - api.load_learning_package(broken_fixture(fixture_name), user=self.user) + api.create_learning_package(broken_fixture(fixture_name), user=self.user) assert publishing_api.LearningPackage.objects.count() == packages_before, ( "A failed restore must not leave anything behind." @@ -411,11 +411,11 @@ def test_unresolved_child(self): def test_unreadable_archive(self): with self.assertRaises(ArchiveNotReadableError): - api.load_learning_package("/no/such/path.zip", user=self.user) + api.create_learning_package("/no/such/path.zip", user=self.user) def test_error_text_lists_every_problem(self): with self.assertRaises(RestoreFailedError) as ctx: - api.load_learning_package(broken_fixture("missing_lp_key"), user=self.user) + api.create_learning_package(broken_fixture("missing_lp_key"), user=self.user) text = ctx.exception.as_text() assert text.startswith("Errors encountered during restore:\n") @@ -431,7 +431,7 @@ class LoadLearningPackageAsDictTest(RestoreTestCase): """ def test_success_shape(self): - result = api.load_learning_package_as_dict( + result = api.load_learning_package( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -451,7 +451,7 @@ def test_success_shape(self): } def test_error_shape(self): - result = api.load_learning_package_as_dict( + result = api.load_learning_package( broken_fixture("missing_lp_key"), user=self.user ) @@ -462,7 +462,7 @@ def test_error_shape(self): assert "package.toml" in result["log_file_error"].getvalue() def test_unreadable_archive_is_also_reported_as_a_dict(self): - result = api.load_learning_package_as_dict("/no/such/path.zip", user=self.user) + result = api.load_learning_package("/no/such/path.zip", user=self.user) assert result["status"] == "error" assert "/no/such/path.zip" in result["log_file_error"].getvalue() From a9134d23bb14680727107a868f2fb7b19326b664 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 16:04:39 -0400 Subject: [PATCH 06/14] feat: accept archives that wrap their contents in a folder Encapsulates extraction in PayloadExtractor and finishes the two TODOs that were sitting in that class. Wrapper-folder archives ----------------------- `zip -r MyLib.zip MyLib` compresses the folder rather than its contents, so the archive has a single top-level directory with package.toml inside it. That is a perfectly reasonable thing to hand us, but we rejected it with "Root Package file not found at expected path". `find_archive_root` now looks one level down: ignore archiving debris (__MACOSX, dotfiles), keep the top-level directories that actually contain a package.toml, and re-root if exactly one qualifies. Re-rooting is done by wrapping the filesystem in a DirFileSystem, so every path expression downstream keeps working unchanged and is unaware anything happened. Requiring package.toml inside the candidate is load-bearing rather than belt-and-braces: payload_test_data/entities/ contains exactly one subdirectory, and a rule based only on "a single top-level folder" would silently re-root into it and break every entity test. There is a named regression test for that. Two details worth recording. The previous sketch used `len(fs.ls('.')) == 1`, which could never have fired -- ls(".") returns [] on a ZipFileSystem, which is the case it was written for; ls("") is what lists the top level. And a zip made with macOS Finder's "Compress" carries __MACOSX and often .DS_Store beside the folder, so any rule counting total top-level entries would fail on the most common way a non-technical user produces an archive. Encapsulation ------------- The module-level extraction functions become methods, with fs held as constructor state and root_package_path as a class attribute. This is what makes the class the seam its docstring always claimed it was, for teams whose archives are laid out differently -- the module docstring cites MIT DELTA, who encode much of the Section/Subsection/Unit hierarchy in a single file. Beyond the mechanical move: * Duplicate detection moves from extract_entity_data up into extract_entities_data. A duplicate is a property of the *set* of files, not of any one file, and this drops an optional parameter that only existed to smuggle the mapping down. * get_collection_file_paths() mirrors get_entity_file_paths(); extract() used to inline the glob, which is exactly the layout knowledge a subclass wants to override. * The "Duplicate collections are a problem too" TODO is resolved, not deferred: schema.CompletePackageInputData.check_for_duplicate_keys already catches them, and can name both files because it has their data by then. Reporting the detected root --------------------------- UnvalidatedLearningPackageInput and ValidatedLearningPackageInput carry the folder we picked, and RestoreFailedError.as_text() emits an "Archive root:" line when there is one. Error paths themselves stay relative to that root, so the "fs:" static-asset pointers, entity_path_mapping and SchemaError.path all live in a single path space. The one thing that had to be got right: extract() returns the *re-rooted* filesystem, because loading.py resolves static assets against the "fs:" pointers written during extraction. Handing back the original would break images for wrapper archives only. Tested directly, both at the payload layer and end to end. Tests ----- 139 -> 165 passing; payload.py reaches 100% coverage, PayloadExtractor having been the last uncovered code in the applet. New FindArchiveRootTest covers package.toml at the top, a single wrapper, macOS debris, a wrapper beside a stray file, a folder with no package.toml, two ambiguous candidates, two levels of nesting and an empty archive -- over both a zip and a directory, since ls("") differs between them. End-to-end tests confirm a wrapper zip and a wrapper directory restore identically to a flat archive. folder_to_zip_path gains prefix and extra_names, so wrapper archives are built at test time from the existing fixture rather than duplicating 20+ files. Co-Authored-By: Claude Opus 5 (1M context) --- .../applets/backup_restore/api.py | 4 +- .../applets/backup_restore/errors.py | 17 +- .../applets/backup_restore/payload.py | 769 ++++++++++-------- .../applets/backup_restore/validation.py | 5 + test_utils/zip_file_utils.py | 18 +- .../applets/backup_restore/test_loading.py | 137 ++++ .../applets/backup_restore/test_payload.py | 268 ++++-- .../applets/backup_restore/test_validation.py | 38 +- 8 files changed, 848 insertions(+), 408 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 9def63331..833150f9d 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -57,14 +57,14 @@ def create_learning_package( Both descend from ``BackupRestoreError``. """ fs = archive.read_fs_for_path(path_str) - unvalidated_input = payload.extract_unvalidated_learning_package(fs) + unvalidated_input = payload.PayloadExtractor(fs).extract() validated_input = validation.validate(unvalidated_input) # Bail out before touching the database. We deliberately don't do a partial # restore: a half-loaded Learning Package is harder to reason about than no # Learning Package at all. if validated_input.errors: - raise RestoreFailedError(validated_input.errors) + raise RestoreFailedError(validated_input.errors, validated_input.root) loader = loading.Loader(validated_input) archive_lp_input = loader.data.learning_package # LearningPackageInputData diff --git a/src/openedx_content/applets/backup_restore/errors.py b/src/openedx_content/applets/backup_restore/errors.py index 9f34d8bb1..491ecb0bf 100644 --- a/src/openedx_content/applets/backup_restore/errors.py +++ b/src/openedx_content/applets/backup_restore/errors.py @@ -188,8 +188,16 @@ class RestoreFailedError(BackupRestoreError): one run at a time. """ - def __init__(self, errors: list[BackupRestoreError]): + def __init__( + self, + errors: list[BackupRestoreError], + archive_root: str | None = None, + ): self.errors = list(errors) + # The folder inside the archive we treated as the root, when the archive + # wrapped its contents in one. Every path below is relative to it, so + # saying so once up front saves a lot of confusion. + self.archive_root = archive_root super().__init__(f"Restore failed with {len(self.errors)} error(s).") def __str__(self): @@ -202,5 +210,8 @@ def as_text(self) -> str: The format matches what the pre-pydantic implementation wrote out, so that existing consumers of the restore log keep working. """ - lines = [str(err) for err in self.errors] - return "Errors encountered during restore:\n" + "\n".join(lines) + "\n" + lines = ["Errors encountered during restore:"] + if self.archive_root: + lines.append(f"Archive root: {self.archive_root}/") + lines.extend(str(err) for err in self.errors) + return "\n".join(lines) + "\n" diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index b2e7c9570..e64b776fc 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -14,6 +14,7 @@ import attrs from fsspec import AbstractFileSystem +from fsspec.implementations.dirfs import DirFileSystem from .errors import ( DuplicateFoundError, @@ -28,6 +29,11 @@ ROOT_PACKAGE_PATH = "package.toml" +# Debris that archiving tools leave at the top level. These shouldn't count when +# we're working out whether an archive wraps its contents in a single folder. +# macOS's "Compress" in particular always adds __MACOSX, and often .DS_Store. +IGNORED_TOP_LEVEL_NAMES = frozenset({"__MACOSX"}) + @attrs.define(frozen=True) class UnvalidatedLearningPackageInput: @@ -45,36 +51,60 @@ class UnvalidatedLearningPackageInput: # Mapping of entity refs to the paths where we found them. entity_path_mapping: dict[str, str] + # The folder inside the archive that we treated as the root, or None if the + # archive's contents were at the top level. Every path in ``raw_data``, + # ``entity_path_mapping`` and ``errors`` is relative to this, so it's only + # useful for telling a human what we decided. + root: str | None = None -class PayloadExtractor: + +def find_archive_root( + fs: AbstractFileSystem, + root_package_path: str = ROOT_PACKAGE_PATH, +) -> str | None: """ - Extracts files from a file system and generates unvalidated input. + Find the folder to treat as the archive root, or None to use ``fs`` as-is. + + People often build an archive by compressing a folder rather than that + folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a single + top-level directory with everything (including package.toml) inside it. That + is a reasonable thing to hand us, so we accept it. - TODO: This is not in use yet, but we want to eventually place extraction- - related functionality in this class so that it's easier to swap out - extraction behavior with other classes later, e.g. if people have different - formatting ideas for their archive formats. + We only look one level down, and we require that the candidate directory + actually contains a ``root_package_path``. That second condition matters more + than it looks: without it, *any* archive whose top level happens to hold a + single directory would be re-rooted into it. + + This function never raises. An archive with no package.toml anywhere returns + None, and the missing file is reported later as an extraction error, which is + where that error belongs. """ + if fs.exists(root_package_path): + return None - def __init__(self, fs: AbstractFileSystem): - self.fs = fs + candidates = [] + # Note: this must be ls("") rather than ls("."), which returns [] on a + # ZipFileSystem -- i.e. exactly the case we're here to handle. + for entry in fs.ls("", detail=False): + name = entry.rsplit("/", 1)[-1] + if name in IGNORED_TOP_LEVEL_NAMES or name.startswith("."): + continue + if not fs.isdir(entry): + continue + if fs.exists(f"{entry}/{root_package_path}"): + candidates.append(entry) - # TODO: This is not complete. The problem we eventually want to solve is - # that sometimes people archive the right thing and package.toml is at - # the root, but sometimes people make an archive that has one folder in - # it, and everything (including the package.toml) is in that folder. So - # we want to gracefully accept that format and have our root change. - if self.fs.exists("package.toml"): - self.root = "" - elif len(fs.ls('.')) == 1: - pass + # More than one candidate is ambiguous, and guessing would be worse than + # saying we couldn't find the file. + if len(candidates) == 1: + return candidates[0] + return None -def extract_unvalidated_learning_package( - fs: AbstractFileSystem, -) -> UnvalidatedLearningPackageInput: + +class PayloadExtractor: """ - Extract the raw, unvalidated Learning Package metadata. + Extracts files from a file system and generates unvalidated input. We scan through the archive and compile a Python dictionary that can be validated against CompletePackageInputData. This mostly involves reading a @@ -88,7 +118,9 @@ def extract_unvalidated_learning_package( different set of conventions. For instance, the MIT Disciplinary Experts in Learning Technology and Applications team prefers to author in a way that encodes large parts of the hierarchy (Section -> Subsection -> Unit) in a - single file, with pointers to certain Components in different files. + single file, with pointers to certain Components in different files. Such a + team would subclass this and override the handful of methods that know about + where files live and how they're shaped. Errors can happen at this layer, but they are errors related to the consistency of the archive payload format itself. So errors that need to be @@ -103,377 +135,406 @@ def extract_unvalidated_learning_package( that are errors here are the things that prevent us from creating a UnvalidatedLearningPackageInput at all. """ - # The general philosophy here is to always march on and get as much as - # possible, even if we know the upload is doomed. - unvalidated: dict = {} - errors: list[ExtractionError] = [] - - # Root Package Metadata - try: - # This adds the "meta" and "learning_package" keys - unvalidated |= extract_root_package_data(fs, ROOT_PACKAGE_PATH) - except ExtractionError as err: - errors.append(err) - - # PublishableEntities & versions (components, units, sections, subsections) - entities_data, entity_path_mapping, entities_errors = extract_entities_data( - fs, get_entity_file_paths(fs) - ) - unvalidated["entities"] = entities_data - errors.extend(entities_errors) - - # Collections - # TODO: Duplicate collections are a problem too. - collections = [] - for collection_file_path in sorted(fs.glob("collections/*.toml")): + + # Declared as a class attribute so that a subclass using a different layout + # can point it somewhere else. + root_package_path = ROOT_PACKAGE_PATH + + def __init__(self, fs: AbstractFileSystem): + self.source_fs = fs + self.root = find_archive_root(fs, self.root_package_path) + + # Re-rooting with a DirFileSystem means nothing below this line has to + # know whether the archive wrapped its contents in a folder: every path + # we read or report is relative to self.fs either way. + self.fs = DirFileSystem(path=self.root, fs=fs) if self.root else fs + + def extract(self) -> UnvalidatedLearningPackageInput: + """ + Read the whole archive, gathering errors rather than raising them. + """ + # The general philosophy here is to always march on and get as much as + # possible, even if we know the upload is doomed. + unvalidated: dict = {} + errors: list[ExtractionError] = [] + + # Root Package Metadata try: - collections.append(extract_collection_data(fs, collection_file_path)) + # This adds the "meta" and "learning_package" keys + unvalidated |= self.extract_root_package_data() except ExtractionError as err: errors.append(err) - unvalidated["collections"] = collections - - return UnvalidatedLearningPackageInput( - raw_data=unvalidated, - errors=errors, - fs=fs, - entity_path_mapping=entity_path_mapping, - ) + # PublishableEntities & versions (components, units, sections, subsections) + entities_data, entity_path_mapping, entities_errors = self.extract_entities_data( + self.get_entity_file_paths() + ) + unvalidated["entities"] = entities_data + errors.extend(entities_errors) + + # Collections. Note that duplicate Collection keys are *not* checked + # here: unlike entities, collections are assembled into a list, so a + # duplicate loses no data at this layer. It's caught during validation by + # CompletePackageInputData.check_for_duplicate_keys, which can give a + # better message because it has both files' data by then. + collections = [] + for collection_file_path in self.get_collection_file_paths(): + try: + collections.append(self.extract_collection_data(collection_file_path)) + except ExtractionError as err: + errors.append(err) + unvalidated["collections"] = collections + + return UnvalidatedLearningPackageInput( + raw_data=unvalidated, + errors=errors, + # This must be the re-rooted filesystem, because the "fs:" static + # asset pointers we write below are relative to it. + fs=self.fs, + entity_path_mapping=entity_path_mapping, + root=self.root, + ) -def extract_root_package_data(fs: AbstractFileSystem, path: str) -> dict: - """ - Extract the "meta" and "learning_package" from the TOML file at path. + def extract_root_package_data(self, path: str | None = None) -> dict: + """ + Extract the "meta" and "learning_package" from the TOML file at path. - This is a straightforward extraction because we don't have to transform the - actual fields in the data. We expect to see a TOML file that looks something - like this: + This is a straightforward extraction because we don't have to transform the + actual fields in the data. We expect to see a TOML file that looks something + like this: - [meta] - format_version = 1 - created_by = "eddy" - created_by_email = "eddy@axim.org" - created_at = 2026-03-11T19:20:20.394360Z - origin_server = "studio.local.openedx.io" + [meta] + format_version = 1 + created_by = "eddy" + created_by_email = "eddy@axim.org" + created_at = 2026-03-11T19:20:20.394360Z + origin_server = "studio.local.openedx.io" - [learning_package] - title = "Fun Library" - key = "lib:Axim:FunLib" - description = "My very fun library! 🐢" - created = 2026-02-11T16:32:47.524556Z - updated = 2026-02-20T16:32:47.524556Z + [learning_package] + title = "Fun Library" + key = "lib:Axim:FunLib" + description = "My very fun library! 🐢" + created = 2026-02-11T16:32:47.524556Z + updated = 2026-02-20T16:32:47.524556Z - The output should look like: + The output should look like: - { - 'meta': { - 'format_version': 1, - 'created_by': 'eddy', - 'created_by_email': 'eddy@axim.org', - 'created_at': datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), - 'origin_server': 'studio.local.openedx.io' - }, - 'learning_package': { - 'title': 'Fun Library', - 'key': 'lib:Axim:FunLib', - 'description': 'My very fun library! 🐢', - 'created': datetime(2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc), - 'updated': datetime(2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc), + { + 'meta': { + 'format_version': 1, + 'created_by': 'eddy', + 'created_by_email': 'eddy@axim.org', + 'created_at': datetime(2026, 3, 11, 19, 20, 20, 394360, tzinfo=timezone.utc), + 'origin_server': 'studio.local.openedx.io' + }, + 'learning_package': { + 'title': 'Fun Library', + 'key': 'lib:Axim:FunLib', + 'description': 'My very fun library! 🐢', + 'created': datetime(2026, 2, 11, 16, 32, 47, 524556, tzinfo=timezone.utc), + 'updated': datetime(2026, 2, 20, 16, 32, 47, 524556, tzinfo=timezone.utc), + } } - } - - We need to return a Python dict that we get from parsing this. Most of this - function is error handling. The error checking at this layer is minimal, and - is mostly focused on making sure that the file exists, is parseable, and has - the two tables we expect it to have. - """ - file_description = "Root Package" - # Check: Root Package file exists at all. - if not fs.exists(path): - raise MissingFileError(file_description, path=path) + We need to return a Python dict that we get from parsing this. Most of this + method is error handling. The error checking at this layer is minimal, and + is mostly focused on making sure that the file exists, is parseable, and has + the two tables we expect it to have. + """ + file_description = "Root Package" + if path is None: + path = self.root_package_path + + # Check: Root Package file exists at all. + if not self.fs.exists(path): + raise MissingFileError(file_description, path=path) + + root_package_dict = self._load_toml(path, file_description) + + # Check: Don't allow top-level fields outside a [table] + self._check_all_fields_in_tables(root_package_dict, file_description, path) + + # Check: The "[meta]" and "[learning_package]" tables are mandatory + if "meta" not in root_package_dict: + raise TableNotFoundError(file_description, table="meta", path=path) + if "learning_package" not in root_package_dict: + raise TableNotFoundError( + file_description, table="learning_package", path=path + ) - # Check: Is it a valid TOML file? - with fs.open(path, "rb") as package_toml_file: - try: - root_package_dict = tomllib.load(package_toml_file) - except tomllib.TOMLDecodeError as dec_err: - raise InvalidTOMLError( - file_description, details=str(dec_err), path=path - ) from dec_err - - # Check: Don't allow top-level fields outside a [table] - _check_all_fields_in_tables(root_package_dict, file_description, path) - - # Check: The "[meta]" and "[learning_package]" tables are mandatory - if "meta" not in root_package_dict: - raise TableNotFoundError(file_description, table="meta", path=path) - if "learning_package" not in root_package_dict: - raise TableNotFoundError(file_description, table="learning_package", path=path) - - # Check: We only support format_version 1, and don't know what to do with - # anything higher. This leaves us some wiggle-room to declare a 1.x version - # that is backwards compatible, i.e. it will reject 2 and higher, but accept - # 1.1, 1.2, etc. - format_version = root_package_dict["meta"].get("format_version") - is_number = isinstance(format_version, (int, float)) and not isinstance( - format_version, bool - ) - if not is_number or format_version >= 2: - raise UnsupportedFormatError( - f"Format version {format_version} is unsupported (only 1 is supported).", - path=path, + # Check: We only support format_version 1, and don't know what to do with + # anything higher. This leaves us some wiggle-room to declare a 1.x version + # that is backwards compatible, i.e. it will reject 2 and higher, but accept + # 1.1, 1.2, etc. + format_version = root_package_dict["meta"].get("format_version") + is_number = isinstance(format_version, (int, float)) and not isinstance( + format_version, bool ) + if not is_number or format_version >= 2: + raise UnsupportedFormatError( + f"Format version {format_version} is unsupported (only 1 is supported).", + path=path, + ) - return root_package_dict - - -def get_entity_file_paths(fs: AbstractFileSystem) -> list[str]: - """ - Find all the PublishableEntity TOML file paths in our archive. - - We expect our entity TOML files to be in the entities directory, but we have - two categories right now: - - * Component TOML: entities/xblock.v1/{component_type}/{component_code} - * Container TOML: entities/{entity_ref} - - This function looks for TOML files in entities/ or any of its subdirs. We - only exclude matches inside the component_version data, to make sure that we - don't accidentally match media files in the unlikely event where people have - TOML files as static assets. - """ - paths = [ - path - for path in fs.glob("entities/**/*.toml") - # Filter out TOML files that are in component media, e.g. static assets: - if "/component_versions/" not in path - ] - return sorted(paths) # Make the ordering deterministic. - + return root_package_dict -def extract_entities_data(fs: AbstractFileSystem, paths: list[str]): - """ - Extract every entity file, collecting errors instead of raising them. + def get_entity_file_paths(self) -> list[str]: + """ + Find all the PublishableEntity TOML file paths in our archive. - Returns a ``(entities_data, entity_path_mapping, errors)`` tuple. The - path mapping lets later stages report errors against the file an entity - came from, which is not derivable from the entity ref. - """ - entities_data: dict[str, dict] = {} - entity_path_mapping: dict[str, str] = {} - errors: list[ExtractionError] = [] - for entity_file_path in paths: - try: - entity_ref, entity_data = extract_entity_data( - fs, entity_file_path, entity_path_mapping - ) - entities_data[entity_ref] = entity_data - entity_path_mapping[entity_ref] = entity_file_path - except ExtractionError as err: - errors.append(err) + We expect our entity TOML files to be in the entities directory, but we have + two categories right now: - return entities_data, entity_path_mapping, errors + * Component TOML: entities/xblock.v1/{component_type}/{component_code} + * Container TOML: entities/{entity_ref} + This method looks for TOML files in entities/ or any of its subdirs. We + only exclude matches inside the component_version data, to make sure that we + don't accidentally match media files in the unlikely event where people have + TOML files as static assets. + """ + paths = [ + path + for path in self.fs.glob("entities/**/*.toml") + # Filter out TOML files that are in component media, e.g. static assets: + if "/component_versions/" not in path + ] + return sorted(paths) # Make the ordering deterministic. + + def get_collection_file_paths(self) -> list[str]: + """ + Find all the Collection TOML file paths in our archive. + """ + return sorted(self.fs.glob("collections/*.toml")) + + def extract_entities_data(self, paths: list[str]): + """ + Extract every entity file, collecting errors instead of raising them. + + Returns a ``(entities_data, entity_path_mapping, errors)`` tuple. The + path mapping lets later stages report errors against the file an entity + came from, which is not derivable from the entity ref. + + Duplicate detection lives here rather than in ``extract_entity_data`` + because it's a property of the *set* of files, not of any one file. + """ + entities_data: dict[str, dict] = {} + entity_path_mapping: dict[str, str] = {} + errors: list[ExtractionError] = [] + for entity_file_path in paths: + try: + entity_ref, entity_data = self.extract_entity_data(entity_file_path) + + # Check: Is it a duplicate of an Entity that has already been + # defined elsewhere in this archive? Without this, the second + # definition would silently overwrite the first, which would be + # baffling to someone assembling an archive by hand. + if entity_ref in entity_path_mapping: + raise DuplicateFoundError( + f"Entity key {entity_ref}", + entity_path_mapping[entity_ref], + entity_file_path, + ) + + entities_data[entity_ref] = entity_data + entity_path_mapping[entity_ref] = entity_file_path + except ExtractionError as err: + errors.append(err) + + return entities_data, entity_path_mapping, errors + + def extract_entity_data(self, path: str) -> tuple[str, dict]: + """ + This extracts raw entity data from an Entity TOML file. + + PublishableEntities can be both Components (XBlock problems, videos, etc.), + as well as Containers like Units, Subsections, and Sections. Some sample + TOML: + + [entity] + can_stand_alone = true + key = "section-9-ac4b9f" + created = 2026-04-08T15:22:12.780012Z + + [entity.draft] + version_num = 2 + + [entity.published] + version_num = 1 + + [entity.container.section] + + # ### Versions + + [[version]] + title = "Section 9" + version_num = 2 + + [version.container] + children = ["week-7-e73782", "subsection-001-e4bbe5"] + + [[version]] + title = "Section 9" + version_num = 1 + + [version.container] + children = ["week-7-e73782"] + + We return a tuple where the first element is the Entity's key + ("section-9-ac4b9f"), and the second is a dict that would look like: -def extract_entity_data( - fs: AbstractFileSystem, path: str, entity_path_mapping: dict[str, str] | None = None -) -> tuple[str, dict]: - """ - This extracts raw entity data from an Entity TOML file. + { + 'can_stand_alone': True, + 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), + 'draft': { + 'version_num': 2 + }, + 'published': { + 'version_num': 1 + }, + 'container': { + 'section': {} + }, + 'versions': [ + { + 'title': 'Section 9', + 'version_num': 2, + 'container': { + 'children': [ + 'week-7-e73782', + 'subsection-001-e4bbe5' + ] + } + }, + { + 'title': 'Section 9', + 'version_num': 1, + 'container': { + 'children': [ + 'week-7-e73782' + ] + } + } + ] + } - PublishableEntities can be both Components (XBlock problems, videos, etc.), - as well as Containers like Units, Subsections, and Sections. Some sample - TOML: + Note some key differences: - [entity] - can_stand_alone = true - key = "section-9-ac4b9f" - created = 2026-04-08T15:22:12.780012Z + 1. The "entity" table elements have been popped out to the top level. + 2. The "version" list has been renamed to "versions" to feel more natural. + 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back as + part of the tuple. This will become a key/value pair in an "entities" + dict that will hold all publishable entity input data. + """ + file_description = "Entity" - [entity.draft] - version_num = 2 + entity_root_dict = self._load_toml(path, file_description) - [entity.published] - version_num = 1 + # Check: Don't allow top-level fields outside a [table] + self._check_all_fields_in_tables(entity_root_dict, file_description, path) - [entity.container.section] + # Check: Does it define a top level "[entity]" table? Note that this can + # pass if they define a sub-table like "[entity.draft]", since the existence + # of "[entity]" is implicit in that case. If we get that far, rely on + # catching it at the validation step (i.e. after payload extraction). + if "entity" not in entity_root_dict: + raise TableNotFoundError(file_description, "entity", path=path) - # ### Versions + # Check: Does it define an Entity key (i.e. entity_ref)? We need to check + # this now because the dict we have to assemble will use these as keys. + entity = entity_root_dict["entity"] + entity_ref = entity.pop("key", None) + if not entity_ref: + raise FieldMissing(file_description, "entity", "key", path) - [[version]] - title = "Section 9" - version_num = 2 + # Note case difference: we're renaming "version" in the TOML to "versions" + # in the data dict we're assembling. + entity["versions"] = entity_root_dict.pop("version", []) + for version in entity["versions"]: + self._add_component_version_media(version, path) - [version.container] - children = ["week-7-e73782", "subsection-001-e4bbe5"] + return entity_ref, entity - [[version]] - title = "Section 9" - version_num = 1 + def extract_collection_data(self, path: str) -> dict: + """ + Extract the contents of a single Collection TOML file. - [version.container] - children = ["week-7-e73782"] + We record the source path on the way out, so that a later duplicate-key + error can name both of the files involved. + """ + file_description = "Collection" - We return a tuple where the first element is the Entity's key - ("section-9-ac4b9f"), and the second is a dict that would look like: + collection_root_dict = self._load_toml(path, file_description) - { - 'can_stand_alone': True, - 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), - 'draft': { - 'version_num': 2 - }, - 'published': { - 'version_num': 1 - }, - 'container': { - 'section': {} - }, - 'versions': [ - { - 'title': 'Section 9', - 'version_num': 2, - 'container': { - 'children': [ - 'week-7-e73782', - 'subsection-001-e4bbe5' - ] - } - }, - { - 'title': 'Section 9', - 'version_num': 1, - 'container': { - 'children': [ - 'week-7-e73782' - ] - } - } - ] - } + self._check_all_fields_in_tables(collection_root_dict, file_description, path) + if "collection" not in collection_root_dict: + raise TableNotFoundError(file_description, table="collection", path=path) - Note some key differences: + collection_data = collection_root_dict["collection"] + collection_data["src_path"] = path - 1. The "entity" table elements have been popped out to the top level. - 2. The "version" list has been renamed to "versions" to feel more natural. - 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back as - part of the tuple. This will become a key/value pair in an "entities" - dict that will hold all publishable entity input data. - """ - file_description = "Entity" - if entity_path_mapping is None: - entity_path_mapping = {} + return collection_data - # Check: Is it a valid TOML file? - with fs.open(path, "rb") as entity_file: - try: - entity_root_dict = tomllib.load(entity_file) - except tomllib.TOMLDecodeError as dec_err: - raise InvalidTOMLError( - file_description, details=str(dec_err), path=path - ) from dec_err - - # Check: Don't allow top-level fields outside a [table] - _check_all_fields_in_tables(entity_root_dict, file_description, path) - - # Check: Does it define a top level "[entity]" table? Note that this can - # pass if they define a sub-table like "[entity.draft]", since the existence - # of "[entity]" is implicit in that case. If we get that far, rely on - # catching it at the validation step (i.e. after payload extraction). - if "entity" not in entity_root_dict: - raise TableNotFoundError(file_description, "entity", path=path) - - # Check: Does it define an Entity key (i.e. entity_ref)? We need to check - # this now because the dict we have to assemble will use these as keys. - entity = entity_root_dict["entity"] - entity_ref = entity.pop("key", None) - if not entity_ref: - raise FieldMissing(file_description, "entity", "key", path) - - # Check: Is it a duplicate of an Entity that has already been defined - # elsewhere in this archive? - if entity_ref in entity_path_mapping: - raise DuplicateFoundError( - f"Entity key {entity_ref}", entity_path_mapping[entity_ref], path - ) + def _add_component_version_media(self, version: dict, entity_path: str) -> None: + """ + Attach a Component version's media, if this version has any on disk. - # Note case difference: we're renaming "version" in the TOML to "versions" - # in the data dict we're assembling. - entity["versions"] = entity_root_dict.pop("version", []) - for version in entity["versions"]: - # Do our best to put together entity version data (and component version - # data), but don't worry about validating the results (that can happen - # during the validation step). + Do our best to put together entity version data (and component version + data), but don't worry about validating the results (that can happen + during the validation step). + """ version_num = version.get("version_num") comp_ver_dir = os.path.join( - os.path.splitext(path)[0], + os.path.splitext(entity_path)[0], "component_versions", f"v{version_num}", ) - if fs.exists(comp_ver_dir): - version["component"] = {} - media = { - os.path.relpath(path, comp_ver_dir): fs.read_text(path) - for path in fs.glob(f"{comp_ver_dir}/*") - if fs.isfile(path) - } - # Any static files are encoded as pointers. - # TODO: Convert this to data-urls later - for static_file_path in fs.glob(f"{comp_ver_dir}/static/**"): - if fs.isfile(static_file_path): - rel_path = os.path.relpath(static_file_path, comp_ver_dir) - media[rel_path] = f"fs:{static_file_path}" + if not self.fs.exists(comp_ver_dir): + return - version["component"]["media"] = media - - return entity_ref, entity - - -def extract_collection_data(fs: AbstractFileSystem, path: str) -> dict: - """ - Extract the contents of a single Collection TOML file. - - We record the source path on the way out, so that a later duplicate-key - error can name both of the files involved. - """ - file_description = "Collection" - - with fs.open(path, "rb") as collection_toml_file: - try: - collection_root_dict = tomllib.load(collection_toml_file) - except tomllib.TOMLDecodeError as dec_err: - raise InvalidTOMLError( - file_description, details=str(dec_err), path=path - ) from dec_err - - _check_all_fields_in_tables(collection_root_dict, file_description, path) - if "collection" not in collection_root_dict: - raise TableNotFoundError( - file_description, table="collection", path=path - ) - - collection_data = collection_root_dict["collection"] - collection_data["src_path"] = path - - return collection_data - - -def _check_all_fields_in_tables(data: dict, file_description, path): - """ - Raise an error if fields are declared outside of a table. - - The convention for our TOML files is that keys are always in a table, so if - it's *not* in a table, that's likely an omission/error that might otherwise - be difficult to catch because they'd be "missing" from the place they're - supposed to be in the parsed data structure, but that wouldn't be obvious to - someone editing the files by hand. - """ - fields_outside_of_tables = [ - field - for field, val in data.items() - if not isinstance(val, dict) and not isinstance(val, list) - ] - if fields_outside_of_tables: - raise FieldsNotInTable( - file_description, fields=fields_outside_of_tables, path=path - ) + media = { + os.path.relpath(media_path, comp_ver_dir): self.fs.read_text(media_path) + for media_path in self.fs.glob(f"{comp_ver_dir}/*") + if self.fs.isfile(media_path) + } + # Any static files are encoded as pointers. + # TODO: Convert this to data-urls later + for static_file_path in self.fs.glob(f"{comp_ver_dir}/static/**"): + if self.fs.isfile(static_file_path): + rel_path = os.path.relpath(static_file_path, comp_ver_dir) + media[rel_path] = f"fs:{static_file_path}" + + version["component"] = {"media": media} + + def _load_toml(self, path: str, file_description: str) -> dict: + """ + Parse the TOML file at ``path``, or raise InvalidTOMLError. + """ + with self.fs.open(path, "rb") as toml_file: + try: + return tomllib.load(toml_file) + except tomllib.TOMLDecodeError as dec_err: + raise InvalidTOMLError( + file_description, details=str(dec_err), path=path + ) from dec_err + + @staticmethod + def _check_all_fields_in_tables(data: dict, file_description: str, path: str): + """ + Raise an error if fields are declared outside of a table. + + The convention for our TOML files is that keys are always in a table, so if + it's *not* in a table, that's likely an omission/error that might otherwise + be difficult to catch because they'd be "missing" from the place they're + supposed to be in the parsed data structure, but that wouldn't be obvious to + someone editing the files by hand. + """ + fields_outside_of_tables = [ + field + for field, val in data.items() + if not isinstance(val, dict) and not isinstance(val, list) + ] + if fields_outside_of_tables: + raise FieldsNotInTable( + file_description, fields=fields_outside_of_tables, path=path + ) diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py index 7908682f2..5423afa5f 100644 --- a/src/openedx_content/applets/backup_restore/validation.py +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -53,6 +53,10 @@ class ValidatedLearningPackageInput: errors: list[BackupRestoreError] + # The folder inside the archive that was treated as its root, if any. Purely + # informational -- every path in ``errors`` is already relative to it. + root: str | None = None + def validate( unvalidated_lp: UnvalidatedLearningPackageInput, @@ -76,6 +80,7 @@ def validate( data=data, fs=unvalidated_lp.fs, errors=errors, + root=unvalidated_lp.root, ) diff --git a/test_utils/zip_file_utils.py b/test_utils/zip_file_utils.py index 64e75a0cb..c9876abcb 100644 --- a/test_utils/zip_file_utils.py +++ b/test_utils/zip_file_utils.py @@ -26,7 +26,13 @@ def folder_to_inmemory_zip(folder_path: str) -> zipfile.ZipFile: return zipfile.ZipFile(buffer, "r") -def folder_to_zip_path(folder_path: str, dest_dir: str, name: str = "archive.zip") -> str: +def folder_to_zip_path( + folder_path: str, + dest_dir: str, + name: str = "archive.zip", + prefix: str = "", + extra_names: tuple = (), +) -> str: """ Write the contents of a folder out as a real zip file on disk. @@ -38,6 +44,11 @@ def folder_to_zip_path(folder_path: str, dest_dir: str, name: str = "archive.zip folder_path (str): Path to the folder to zip. dest_dir (str): Directory to write the zip file into. name (str): File name to give the zip file. + prefix (str): Prepended to every archive member, e.g. ``"MyLib/"``. Use + this to build the kind of archive you get from ``zip -r x.zip MyLib``, + where everything sits inside a single wrapper folder. + extra_names (tuple): Extra (empty) members to add, for simulating the + debris real archiving tools leave behind, e.g. ``"__MACOSX/._MyLib"``. Returns: str: The path of the zip file that was written. @@ -47,5 +58,8 @@ def folder_to_zip_path(folder_path: str, dest_dir: str, name: str = "archive.zip with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: for file_path in sorted(folder.rglob("*")): if file_path.is_file(): - zipf.write(file_path, arcname=str(file_path.relative_to(folder))) + arcname = prefix + str(file_path.relative_to(folder)) + zipf.write(file_path, arcname=arcname) + for extra_name in extra_names: + zipf.writestr(extra_name, b"") return str(zip_path) diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py index db79b784a..395fdec50 100644 --- a/tests/openedx_content/applets/backup_restore/test_loading.py +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -16,6 +16,7 @@ * Its entities cover every draft/published combination we care about. """ import os +import shutil import tempfile from datetime import datetime, timezone from io import StringIO @@ -577,3 +578,139 @@ def test_str_lists_the_individual_errors(self): error = RestoreFailedError([MissingFileError("Root Package", path="package.toml")]) assert str(error) == error.as_text() assert "Root Package file not found" in str(error) + + +class RestoreWrapperArchiveTest(RestoreTestCase): + """ + Archives that wrap their contents in a single folder. + + ``zip -r MyLib.zip MyLib`` compresses the folder rather than its contents, + which is a perfectly reasonable thing to hand us. Such an archive must + restore identically to a flat one. + """ + + def as_wrapper_dir(self, folder: str, wrapper: str = "MyLib") -> str: + """Copy a fixture folder one level down inside a fresh temp directory.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + shutil.copytree(folder, os.path.join(tmp_dir.name, wrapper)) + return tmp_dir.name + + def comparable(self, result): + """Everything about a restore except what differs by construction.""" + data = dict(vars(result.lp_restored_data)) + del data["id"] + del data["package_ref"] + return data + + def test_wrapper_zip_matches_flat_archive(self): + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + wrapped_zip = folder_to_zip_path( + self.fixtures_folder, tmp_dir.name, prefix="MyLib/" + ) + + flat = api.create_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib:flat:one" + ) + wrapped = api.create_learning_package( + wrapped_zip, user=self.user, package_ref="lib:wrapped:zip" + ) + + assert self.comparable(wrapped) == self.comparable(flat) + assert vars(wrapped.backup_metadata) == vars(flat.backup_metadata) + + def test_wrapper_directory_matches_flat_archive(self): + flat = api.create_learning_package( + self.fixtures_folder, user=self.user, package_ref="lib:flat:two" + ) + wrapped = api.create_learning_package( + self.as_wrapper_dir(self.fixtures_folder), + user=self.user, + package_ref="lib:wrapped:dir", + ) + + assert self.comparable(wrapped) == self.comparable(flat) + + def test_macos_style_zip(self): + """A zip made with Finder's "Compress" carries __MACOSX and .DS_Store.""" + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + mac_zip = folder_to_zip_path( + self.fixtures_folder, + tmp_dir.name, + prefix="MyLib/", + extra_names=("__MACOSX/._MyLib", ".DS_Store"), + ) + + result = api.create_learning_package( + mac_zip, user=self.user, package_ref="lib:wrapped:mac" + ) + + assert result.status == "success" + assert result.lp_restored_data.num_components == 7 + + def test_static_assets_survive_a_wrapper(self): + """ + Static files still resolve when the archive is wrapped. + + The ``fs:`` pointers written during extraction are relative to the + re-rooted filesystem, so this is what catches the case where the wrong + filesystem gets handed downstream -- it would break images for wrapper + archives only. + """ + tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(tmp_dir.cleanup) + wrapped_zip = folder_to_zip_path( + self.fixtures_folder, tmp_dir.name, prefix="MyLib/" + ) + + result = api.create_learning_package( + wrapped_zip, user=self.user, package_ref="lib:wrapped:static" + ) + + component = components_api.get_components(result.lp_restored_data.id).get( + publishable_entity__entity_ref=( + "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" + ) + ) + draft = publishing_api.get_draft_version(component.publishable_entity.id) + by_path = { + cvm.path: cvm.media + for cvm in draft.componentversion.componentversionmedia_set.all() + } + assert set(by_path) == {"block.xml", "static/me.png"} + assert by_path["static/me.png"].has_file + assert by_path["static/me.png"].read_file().read() + + def test_ambiguous_archive_is_refused(self): + """ + Several candidate folders means we don't guess. + + ``fixtures/broken/`` holds a handful of directories that each contain a + package.toml, so there is no single obvious root. + """ + packages_before = publishing_api.LearningPackage.objects.count() + + with self.assertRaises(RestoreFailedError) as ctx: + api.create_learning_package( + os.path.join(FIXTURES_DIR, "broken"), user=self.user + ) + + assert publishing_api.LearningPackage.objects.count() == packages_before + assert any( + isinstance(err, MissingFileError) for err in ctx.exception.errors + ) + + def test_detected_root_is_reported_in_the_error_text(self): + """ + When we do re-root, say so -- every path we report is relative to it. + """ + wrapper_dir = self.as_wrapper_dir(broken_fixture("missing_lp_key")) + + with self.assertRaises(RestoreFailedError) as ctx: + api.create_learning_package(wrapper_dir, user=self.user) + + text = ctx.exception.as_text() + assert "Archive root: MyLib/" in text + assert "package.toml: learning_package.key" in text diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py index 8c26b4ea1..2f2084f8e 100644 --- a/tests/openedx_content/applets/backup_restore/test_payload.py +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -6,20 +6,24 @@ This module tests our ability to extract data from the backup archive TOML files and resources, and assemble them into a combined document that represents the entire LearningPackage, and is encapsulated in UnvalidatedLearningPackageInput. -Most of these test functions that examine individual files. The functions in -payload.py were designed to mostly accept an AbstractFileSystem and path as -arguments, so it should be possible to do simple test calls on TOML files and -dirs without having to mock anything. +Most of these test methods that examine individual files. PayloadExtractor +takes the filesystem once, at construction, and its methods take a path, so it +should be possible to do simple test calls on TOML files and dirs without having +to mock anything. These tests are strictly for the payload module, and therefore don't need Django to run. """ +import io +import tempfile +import zipfile from datetime import datetime, timezone from pathlib import Path from unittest import TestCase from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.zip import ZipFileSystem from openedx_content.applets.backup_restore import payload @@ -34,38 +38,40 @@ class ExtractRootPackageFileTest(TestCase): def setUpClass(cls): super().setUpClass() cls.fs = DirFileSystem(TEST_DATA_ROOT / "root_packages") + cls.extractor = payload.PayloadExtractor(cls.fs) @classmethod def tearDownClass(cls): del cls.fs + del cls.extractor super().tearDownClass() def test_file_not_found(self): with self.assertRaises(payload.MissingFileError) as ctx: - payload.extract_root_package_data(self.fs, "does_not_exist.toml") + self.extractor.extract_root_package_data("does_not_exist.toml") assert ctx.exception.path == "does_not_exist.toml" def test_broken_toml(self): with self.assertRaises(payload.InvalidTOMLError) as ctx: - payload.extract_root_package_data(self.fs, "broken.toml") + self.extractor.extract_root_package_data("broken.toml") assert ctx.exception.path == "broken.toml" def test_fields_not_in_table(self): with self.assertRaises(payload.FieldsNotInTable) as ctx: - payload.extract_root_package_data(self.fs, "fields_not_in_table.toml") + self.extractor.extract_root_package_data("fields_not_in_table.toml") assert ctx.exception.path == "fields_not_in_table.toml" assert ctx.exception.fields == ["created_by", "format_version"] def test_missing_meta_table(self): with self.assertRaises(payload.TableNotFoundError) as ctx: - payload.extract_root_package_data(self.fs, "missing_meta.toml") + self.extractor.extract_root_package_data("missing_meta.toml") assert ctx.exception.path == "missing_meta.toml" assert ctx.exception.table == "meta" assert "[meta]" in str(ctx.exception) def test_missing_learning_package_table(self): with self.assertRaises(payload.TableNotFoundError) as ctx: - payload.extract_root_package_data(self.fs, "missing_learning_package.toml") + self.extractor.extract_root_package_data("missing_learning_package.toml") assert ctx.exception.path == "missing_learning_package.toml" assert ctx.exception.table == "learning_package" assert "[learning_package]" in str(ctx.exception) @@ -73,37 +79,27 @@ def test_missing_learning_package_table(self): def test_unsupported_format_version(self): # We don't support format_version=2 with self.assertRaises(payload.UnsupportedFormatError): - payload.extract_root_package_data( - self.fs, "unsupported_format_version_2.toml" - ) + self.extractor.extract_root_package_data("unsupported_format_version_2.toml") # We don't support format_version as anthing other than number with self.assertRaises(payload.UnsupportedFormatError): - payload.extract_root_package_data( - self.fs, "unsupported_format_version_b.toml" - ) + self.extractor.extract_root_package_data("unsupported_format_version_b.toml") # ...and a boolean is not a number, even though Python's bool is a # subclass of int and would otherwise read as version 1. with self.assertRaises(payload.UnsupportedFormatError): - payload.extract_root_package_data( - self.fs, "unsupported_format_version_true.toml" - ) + self.extractor.extract_root_package_data("unsupported_format_version_true.toml") # We will allow format_version 1.x though, in case we want to extend our # format in a fully backwards compatible way. - root_data = payload.extract_root_package_data( - self.fs, "unsupported_format_version_1_1.toml" - ) + root_data = self.extractor.extract_root_package_data("unsupported_format_version_1_1.toml") assert root_data["meta"]["format_version"] == 1.1 def test_ignore_unknown_tables(self): """Allow for forwards compatibility.""" - assert "unknown" in payload.extract_root_package_data( - self.fs, "unknown_table.toml" - ) + assert "unknown" in self.extractor.extract_root_package_data("unknown_table.toml") def test_minimal(self): - data = payload.extract_root_package_data(self.fs, "minimal.toml") + data = self.extractor.extract_root_package_data("minimal.toml") assert data == { "meta": { "format_version": 1, @@ -112,7 +108,7 @@ def test_minimal(self): } def test_normal(self): - data = payload.extract_root_package_data(self.fs, "normal_ulmo_v1.toml") + data = self.extractor.extract_root_package_data("normal_ulmo_v1.toml") assert data == { "meta": { "format_version": 1, @@ -144,27 +140,29 @@ class ExtractEntityDataTest(TestCase): def setUpClass(cls): super().setUpClass() cls.fs = DirFileSystem(TEST_DATA_ROOT / "entities") + cls.extractor = payload.PayloadExtractor(cls.fs) @classmethod def tearDownClass(cls): del cls.fs + del cls.extractor super().tearDownClass() def test_broken_toml(self): with self.assertRaises(payload.InvalidTOMLError) as ctx: - payload.extract_entity_data(self.fs, "broken.toml") + self.extractor.extract_entity_data("broken.toml") assert ctx.exception.path == "broken.toml" def test_missing_entity_table(self): with self.assertRaises(payload.TableNotFoundError) as ctx: - payload.extract_entity_data(self.fs, "missing_entity_table.toml") + self.extractor.extract_entity_data("missing_entity_table.toml") assert ctx.exception.path == "missing_entity_table.toml" assert ctx.exception.table == "entity" assert "[entity]" in str(ctx.exception) def test_missing_entity_key(self): with self.assertRaises(payload.FieldMissing) as ctx: - payload.extract_entity_data(self.fs, "missing_entity_key.toml") + self.extractor.extract_entity_data("missing_entity_key.toml") assert ctx.exception.missing_field == "key" assert ctx.exception.table == "entity" @@ -182,7 +180,7 @@ def test_dupes(self): extract_entity_data(). """ paths = ["dupe_1.toml", "dupe_2.toml"] - data, _path_mapping, errors = payload.extract_entities_data(self.fs, paths) + data, _path_mapping, errors = self.extractor.extract_entities_data(paths) assert "dupe-key" in data # The first one should have succeeded... assert len(data) == 1 # but the duplicate never made it in. assert len(errors) == 1 # There should be only one error. @@ -199,7 +197,7 @@ def test_ignore_unknown_tables(self): can add attributes without older code choking on them. Unknown tables at the top level are not part of the entity, so they don't come along. """ - _ref, data = payload.extract_entity_data(self.fs, "unknown_table.toml") + _ref, data = self.extractor.extract_entity_data("unknown_table.toml") assert data["future_thing"] == {"some_setting": "hello"} assert "future_top_level" not in data @@ -210,13 +208,13 @@ def test_missing_versions(self): Whether that's actually loadable is the validation step's problem, not ours -- our job is only to faithfully report what's in the file. """ - ref, data = payload.extract_entity_data(self.fs, "missing_versions.toml") + ref, data = self.extractor.extract_entity_data("missing_versions.toml") assert ref == "no-versions-c0ffee" assert data["versions"] == [] assert data["container"] == {"unit": {}} def test_normal_component(self): - ref, data = payload.extract_entity_data(self.fs, "normal_component.toml") + ref, data = self.extractor.extract_entity_data("normal_component.toml") assert ref == "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" assert data["can_stand_alone"] is True assert data["created"] == datetime( @@ -245,7 +243,7 @@ def test_normal_component(self): ) def test_normal_container(self): - ref, data = payload.extract_entity_data(self.fs, "normal_container.toml") + ref, data = self.extractor.extract_entity_data("normal_container.toml") assert ref == "section-9-ac4b9f" assert data == { 'can_stand_alone': True, @@ -290,32 +288,34 @@ class ExtractCollectionDataTest(TestCase): def setUpClass(cls): super().setUpClass() cls.fs = DirFileSystem(TEST_DATA_ROOT / "collections") + cls.extractor = payload.PayloadExtractor(cls.fs) @classmethod def tearDownClass(cls): del cls.fs + del cls.extractor super().tearDownClass() def test_broken_toml(self): with self.assertRaises(payload.InvalidTOMLError) as ctx: - payload.extract_collection_data(self.fs, "broken.toml") + self.extractor.extract_collection_data("broken.toml") assert ctx.exception.path == "broken.toml" def test_fields_not_in_table(self): with self.assertRaises(payload.FieldsNotInTable) as ctx: - payload.extract_collection_data(self.fs, "fields_not_in_table.toml") + self.extractor.extract_collection_data("fields_not_in_table.toml") assert ctx.exception.path == "fields_not_in_table.toml" assert ctx.exception.fields == ["key", "title"] def test_missing_collection_table(self): with self.assertRaises(payload.TableNotFoundError) as ctx: - payload.extract_collection_data(self.fs, "missing_collection_table.toml") + self.extractor.extract_collection_data("missing_collection_table.toml") assert ctx.exception.path == "missing_collection_table.toml" assert ctx.exception.table == "collection" assert "[collection]" in str(ctx.exception) def test_normal(self): - data = payload.extract_collection_data(self.fs, "normal.toml") + data = self.extractor.extract_collection_data("normal.toml") assert data == { "title": "Difficult Problems", "key": "difficult-problems", @@ -339,8 +339,8 @@ def test_dupes_are_not_caught_here(self): a list. Nothing is lost at this layer, so we extract both and let CompletePackageInputData.check_for_duplicate_keys reject them. """ - dupe_1 = payload.extract_collection_data(self.fs, "dupe_1.toml") - dupe_2 = payload.extract_collection_data(self.fs, "dupe_2.toml") + dupe_1 = self.extractor.extract_collection_data("dupe_1.toml") + dupe_2 = self.extractor.extract_collection_data("dupe_2.toml") assert dupe_1["key"] == dupe_2["key"] == "dupe-collection-key" assert dupe_1["src_path"] != dupe_2["src_path"] @@ -356,7 +356,7 @@ class ExtractUnvalidatedLearningPackageTest(TestCase): def test_normal(self): fs = DirFileSystem(FIXTURES_ROOT / "library_backup") - unvalidated = payload.extract_unvalidated_learning_package(fs) + unvalidated = payload.PayloadExtractor(fs).extract() assert unvalidated.errors == [] assert unvalidated.raw_data["learning_package"]["key"] == "lib:WGU:LIB_C001" @@ -373,7 +373,7 @@ def test_entity_path_mapping_uses_declared_key(self): collisions. """ fs = DirFileSystem(FIXTURES_ROOT / "library_backup") - unvalidated = payload.extract_unvalidated_learning_package(fs) + unvalidated = payload.PayloadExtractor(fs).extract() assert ( unvalidated.entity_path_mapping["section1-8ca126"] @@ -385,7 +385,7 @@ def test_entity_path_mapping_uses_declared_key(self): def test_missing_root_package_is_collected_not_raised(self): fs = DirFileSystem(TEST_DATA_ROOT / "empty_archive") - unvalidated = payload.extract_unvalidated_learning_package(fs) + unvalidated = payload.PayloadExtractor(fs).extract() assert len(unvalidated.errors) == 1 error = unvalidated.errors[0] @@ -398,7 +398,7 @@ def test_missing_root_package_is_collected_not_raised(self): def test_duplicate_entities_are_collected(self): fs = DirFileSystem(TEST_DATA_ROOT / "duplicate_entities") - unvalidated = payload.extract_unvalidated_learning_package(fs) + unvalidated = payload.PayloadExtractor(fs).extract() duplicate_errors = [ err for err in unvalidated.errors @@ -415,14 +415,14 @@ def test_static_assets_are_not_mistaken_for_entities(self): TOML files under component_versions/ are static assets, not entities. """ fs = DirFileSystem(FIXTURES_ROOT / "library_backup") - paths = payload.get_entity_file_paths(fs) + paths = payload.PayloadExtractor(fs).get_entity_file_paths() assert paths == sorted(paths) # deterministic ordering assert all("/component_versions/" not in path for path in paths) def test_collection_errors_are_collected(self): fs = DirFileSystem(TEST_DATA_ROOT / "broken_collection") - unvalidated = payload.extract_unvalidated_learning_package(fs) + unvalidated = payload.PayloadExtractor(fs).extract() assert len(unvalidated.errors) == 1 error = unvalidated.errors[0] @@ -431,3 +431,179 @@ def test_collection_errors_are_collected(self): # The rest of the archive still came through. assert unvalidated.raw_data["learning_package"]["key"] == "lib:Axim:FunLib" + + +def dir_fs_with(tmp_path, layout: dict) -> DirFileSystem: + """ + Build a throwaway directory tree and return a filesystem over it. + + ``layout`` maps relative paths to file contents. + """ + for rel_path, contents in layout.items(): + target = Path(tmp_path) / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents) + return DirFileSystem(tmp_path) + + +def zip_fs_with(names) -> ZipFileSystem: + """Build an in-memory zip containing ``names``, and return a filesystem over it.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as zipf: + for name in names: + zipf.writestr(name, "[meta]\nformat_version = 1\n") + buffer.seek(0) + return ZipFileSystem(fo=buffer, mode="r") + + +class FindArchiveRootTest(TestCase): + """ + Tests for accepting archives that wrap their contents in a single folder. + + People routinely build an archive with ``zip -r MyLib.zip MyLib`` rather than + compressing the folder's *contents*, and the result is perfectly sensible to + them. We accept it rather than reporting a missing package.toml. + + Both a zip and a plain directory are covered, because fsspec's ``ls("")`` + behaves differently on each. + """ + + PACKAGE = "[meta]\nformat_version = 1\n" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = self._tmp.name + + def test_package_at_top_level_zip(self): + fs = zip_fs_with(["package.toml", "entities/unit1.toml"]) + assert payload.find_archive_root(fs) is None + + def test_package_at_top_level_dir(self): + fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) + assert payload.find_archive_root(fs) is None + + def test_single_wrapper_folder_zip(self): + fs = zip_fs_with(["MyLib/package.toml", "MyLib/entities/unit1.toml"]) + assert payload.find_archive_root(fs) == "MyLib" + + def test_single_wrapper_folder_dir(self): + fs = dir_fs_with(self.tmp_path, {"MyLib/package.toml": self.PACKAGE}) + assert payload.find_archive_root(fs) == "MyLib" + + def test_macos_style_zip(self): + """ + macOS's "Compress" adds __MACOSX and often .DS_Store beside the folder. + + This is the single most common way a non-technical user produces a zip, + so a rule that just counted top-level entries would fail on most of them. + """ + fs = zip_fs_with([ + "MyLib/package.toml", + "MyLib/entities/unit1.toml", + "__MACOSX/._MyLib", + ".DS_Store", + ]) + assert payload.find_archive_root(fs) == "MyLib" + + def test_wrapper_folder_beside_a_stray_file(self): + fs = zip_fs_with(["MyLib/package.toml", "README.txt"]) + assert payload.find_archive_root(fs) == "MyLib" + + def test_folder_without_a_package_toml_is_not_a_root(self): + """ + Requiring package.toml inside the candidate is what makes this safe. + + Without that check, any archive whose top level happened to hold a single + directory would be re-rooted into it. + """ + fs = zip_fs_with(["MyLib/entities/unit1.toml"]) + assert payload.find_archive_root(fs) is None + + def test_two_candidate_folders_are_ambiguous(self): + fs = zip_fs_with(["LibA/package.toml", "LibB/package.toml"]) + assert payload.find_archive_root(fs) is None + + def test_nested_two_levels_is_not_followed(self): + """We only look one level down; deeper nesting isn't worth guessing at.""" + fs = zip_fs_with(["Outer/MyLib/package.toml"]) + assert payload.find_archive_root(fs) is None + + def test_empty_archive(self): + fs = dir_fs_with(self.tmp_path, {}) + assert payload.find_archive_root(fs) is None + + def test_entities_fixture_is_not_re_rooted(self): + """ + Regression guard for the fixture that nearly broke this. + + ``payload_test_data/entities/`` holds exactly one subdirectory, + ``normal_component/``. A rule based on "a single top-level folder" would + silently re-root into it and break every entity test in this module. + """ + fs = DirFileSystem(TEST_DATA_ROOT / "entities") + assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor(fs).root is None + + +class ExtractThroughWrapperTest(TestCase): + """ + Extracting a wrapper-style archive gives the same result as a flat one. + + Everything the extractor reports is relative to the detected root, so the + only observable difference should be ``root`` itself. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + flat_fs = DirFileSystem(FIXTURES_ROOT / "library_backup") + cls.flat = payload.PayloadExtractor(flat_fs).extract() + + # The same fixture, but nested one level down inside "MyLib/". + wrapped_fs = DirFileSystem(FIXTURES_ROOT) + cls.wrapped_extractor = payload.PayloadExtractor(wrapped_fs) + + def test_flat_archive_has_no_root(self): + assert self.flat.root is None + + def test_wrapper_root_is_detected(self): + """ + ``fixtures/`` holds library_backup (with a package.toml) and broken/ + (without one), so library_backup is the only candidate. + """ + assert self.wrapped_extractor.root == "library_backup" + + def test_raw_data_matches_the_flat_archive(self): + wrapped = self.wrapped_extractor.extract() + + assert wrapped.errors == [] + assert wrapped.raw_data == self.flat.raw_data + + def test_paths_are_relative_to_the_detected_root(self): + wrapped = self.wrapped_extractor.extract() + + assert wrapped.entity_path_mapping == self.flat.entity_path_mapping + assert ( + wrapped.entity_path_mapping["section1-8ca126"] + == "entities/section1-extra-8ca126.toml" + ) + + def test_static_asset_pointers_resolve_against_the_returned_fs(self): + """ + The ``fs:`` pointers are relative to the *re-rooted* filesystem. + + This is the subtle one: if extract() returned the original filesystem + instead of the re-rooted one, these pointers would not resolve, and + static assets would silently go missing for wrapper archives only. + """ + wrapped = self.wrapped_extractor.extract() + entity = wrapped.raw_data["entities"][ + "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" + ] + version = next(v for v in entity["versions"] if v["version_num"] == 5) + pointer = version["component"]["media"]["static/me.png"] + + assert pointer.startswith("fs:") + assert wrapped.fs.read_bytes(pointer.removeprefix("fs:")) diff --git a/tests/openedx_content/applets/backup_restore/test_validation.py b/tests/openedx_content/applets/backup_restore/test_validation.py index 57586ff03..5f881b062 100644 --- a/tests/openedx_content/applets/backup_restore/test_validation.py +++ b/tests/openedx_content/applets/backup_restore/test_validation.py @@ -56,7 +56,9 @@ def version(version_num: int, children=None, **overrides) -> dict: return raw -def unvalidated(entities=None, collections=None, errors=None, path_mapping=None, **overrides): +def unvalidated( + entities=None, collections=None, errors=None, path_mapping=None, root=None, **overrides +): """Build an UnvalidatedLearningPackageInput without going through files.""" raw_data = { "meta": {"format_version": 1}, @@ -70,6 +72,7 @@ def unvalidated(entities=None, collections=None, errors=None, path_mapping=None, errors=errors or [], fs=DirFileSystem(FIXTURES_ROOT), entity_path_mapping=path_mapping or {}, + root=root, ) @@ -353,3 +356,36 @@ def test_collection_without_a_src_path(self): def test_schema_error_without_a_location(self): error = SchemaError("something went wrong", path="package.toml") assert str(error) == "package.toml: something went wrong" + + +class ArchiveRootPassthroughTest(TestCase): + """ + The detected archive root travels with the validated input. + + It has no effect on validation -- every path is already relative to it -- but + the error report says which folder we picked, since that isn't obvious from + the paths alone. + """ + + def test_root_is_carried_through(self): + assert validation.validate(unvalidated(root="MyLib")).root == "MyLib" + + def test_no_root_by_default(self): + assert validation.validate(unvalidated()).root is None + + def test_as_text_names_the_root_when_there_is_one(self): + error = RestoreFailedError( + [MissingFileError("Root Package", path="package.toml")], + archive_root="MyLib", + ) + + assert error.as_text() == ( + "Errors encountered during restore:\n" + "Archive root: MyLib/\n" + "package.toml: Root Package file not found at expected path\n" + ) + + def test_as_text_omits_the_root_when_there_isn_t_one(self): + error = RestoreFailedError([MissingFileError("Root Package", path="package.toml")]) + + assert "Archive root" not in error.as_text() From c21b85d79b6a945df9a37bb9bd66f9e324b06096 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 16:56:58 -0400 Subject: [PATCH 07/14] temp: rename to avoid publishing package api collision --- .../applets/backup_restore/api.py | 6 +-- .../applets/backup_restore/payload.py | 10 ++--- .../management/commands/lp_load.py | 4 +- .../applets/backup_restore/test_backup.py | 2 +- .../applets/backup_restore/test_loading.py | 40 +++++++++---------- .../applets/collections/test_api.py | 6 +-- .../applets/collections/test_signals.py | 2 +- .../applets/publishing/test_signals.py | 36 ++++++++--------- 8 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 833150f9d..5a6e4ac99 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -19,12 +19,12 @@ __all__ = [ "create_zip_file", - "create_learning_package", + "load_learning_package_from_path", "load_learning_package", ] -def create_learning_package( +def load_learning_package_from_path( path_str: str, user: UserType, package_ref: str | None = None, @@ -101,7 +101,7 @@ def load_learning_package( ``BackupRestoreError``, so that this can eventually go away. """ try: - result = create_learning_package(path_str, user, package_ref) + result = load_learning_package_from_path(path_str, user, package_ref) except RestoreFailedError as err: return asdict( RestoreResult(status="error", log_file_error=StringIO(err.as_text())) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index e64b776fc..bc759d9e9 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -144,9 +144,9 @@ def __init__(self, fs: AbstractFileSystem): self.source_fs = fs self.root = find_archive_root(fs, self.root_package_path) - # Re-rooting with a DirFileSystem means nothing below this line has to - # know whether the archive wrapped its contents in a folder: every path - # we read or report is relative to self.fs either way. + # Re-rooting with a DirFileSystem means that nothing below this line has + # to know whether the archive wrapped its contents in a folder: every + # path we read or report is relative to self.fs either way. self.fs = DirFileSystem(path=self.root, fs=fs) if self.root else fs def extract(self) -> UnvalidatedLearningPackageInput: @@ -174,8 +174,8 @@ def extract(self) -> UnvalidatedLearningPackageInput: # Collections. Note that duplicate Collection keys are *not* checked # here: unlike entities, collections are assembled into a list, so a - # duplicate loses no data at this layer. It's caught during validation by - # CompletePackageInputData.check_for_duplicate_keys, which can give a + # duplicate loses no data at this layer. It's caught during validation + # by CompletePackageInputData.check_for_duplicate_keys, which can give a # better message because it has both files' data by then. collections = [] for collection_file_path in self.get_collection_file_paths(): diff --git a/src/openedx_content/management/commands/lp_load.py b/src/openedx_content/management/commands/lp_load.py index 2bcd47e86..2c5eef075 100644 --- a/src/openedx_content/management/commands/lp_load.py +++ b/src/openedx_content/management/commands/lp_load.py @@ -8,7 +8,7 @@ from django.core.management import CommandError from django.core.management.base import BaseCommand -from openedx_content.applets.backup_restore.api import create_learning_package +from openedx_content.applets.backup_restore.api import load_learning_package_from_path from openedx_content.applets.backup_restore.errors import BackupRestoreError, RestoreFailedError logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def handle(self, *args, **options): start_time = time.time() try: - result = create_learning_package(path, user=user, package_ref=package_ref) + result = load_learning_package_from_path(path, user=user, package_ref=package_ref) except RestoreFailedError as exc: # The archive is bad. Show every problem we found, not just the first. raise CommandError(exc.as_text()) from exc diff --git a/tests/openedx_content/applets/backup_restore/test_backup.py b/tests/openedx_content/applets/backup_restore/test_backup.py index 1c9e5cf8b..aba0cabdb 100644 --- a/tests/openedx_content/applets/backup_restore/test_backup.py +++ b/tests/openedx_content/applets/backup_restore/test_backup.py @@ -50,7 +50,7 @@ def setUpTestData(cls): ) # Create a Learning Package for the test - cls.learning_package = api.create_learning_package( + cls.learning_package = api.load_learning_package_from_path( package_ref="ComponentTestCase-test-key", title="Components Test Case Learning Package", description="This is a test learning package for components.", diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py index 395fdec50..1a0404b6d 100644 --- a/tests/openedx_content/applets/backup_restore/test_loading.py +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -77,7 +77,7 @@ class RestoreLearningPackageTest(RestoreTestCase): """Restoring a well-formed archive.""" def test_restore_with_explicit_package_ref(self): - result = api.create_learning_package( + result = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -103,7 +103,7 @@ def test_restore_with_explicit_package_ref(self): assert lp is not None, "Learning package was not restored." def test_learning_package_fields_come_from_the_archive(self): - result = api.create_learning_package( + result = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -114,7 +114,7 @@ def test_learning_package_fields_come_from_the_archive(self): def test_restore_with_staged_package_ref(self): """Without an explicit ref we generate one namespaced to the user.""" - result = api.create_learning_package(self.fixtures_folder, user=self.user) + result = api.load_learning_package_from_path(self.fixtures_folder, user=self.user) assert result.status == "success" restored_ref = result.lp_restored_data.package_ref @@ -127,7 +127,7 @@ def test_restore_with_staged_package_ref(self): assert lp is not None, "Learning package with staged ref was not restored." def test_backup_metadata(self): - result = api.create_learning_package(self.fixtures_folder, user=self.user) + result = api.load_learning_package_from_path(self.fixtures_folder, user=self.user) assert result.status == "success" assert result.backup_metadata.format_version == 1 @@ -145,10 +145,10 @@ def test_restore_from_zip_matches_restore_from_directory(self): Reading directly from a directory is new -- the old implementation only accepted zip files -- so it's worth pinning that the two agree. """ - from_dir = api.create_learning_package( + from_dir = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib:from:dir" ) - from_zip = api.create_learning_package( + from_zip = api.load_learning_package_from_path( self.as_zip(self.fixtures_folder), user=self.user, package_ref="lib:from:zip" ) @@ -173,7 +173,7 @@ def test_blank_container_title(self): The ``library_backup`` fixture's ``unit1`` deliberately has a blank title to exercise this path. """ - result = api.create_learning_package( + result = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) @@ -193,7 +193,7 @@ def test_entity_key_need_not_match_filename(self): so the two routinely differ. Two fixture files are named to make sure we don't accidentally start trusting the filename. """ - result = api.create_learning_package( + result = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) lp_id = result.lp_restored_data.id @@ -213,7 +213,7 @@ class RestoreContentTest(RestoreTestCase): def setUp(self): super().setUp() - result = api.create_learning_package( + result = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib-xx:WGU:LIB_C001" ) self.lp = publishing_api.LearningPackage.objects.get( @@ -368,7 +368,7 @@ def assert_refuses(self, fixture_name, error_type): packages_before = publishing_api.LearningPackage.objects.count() with self.assertRaises(RestoreFailedError) as ctx: - api.create_learning_package(broken_fixture(fixture_name), user=self.user) + api.load_learning_package_from_path(broken_fixture(fixture_name), user=self.user) assert publishing_api.LearningPackage.objects.count() == packages_before, ( "A failed restore must not leave anything behind." @@ -412,11 +412,11 @@ def test_unresolved_child(self): def test_unreadable_archive(self): with self.assertRaises(ArchiveNotReadableError): - api.create_learning_package("/no/such/path.zip", user=self.user) + api.load_learning_package_from_path("/no/such/path.zip", user=self.user) def test_error_text_lists_every_problem(self): with self.assertRaises(RestoreFailedError) as ctx: - api.create_learning_package(broken_fixture("missing_lp_key"), user=self.user) + api.load_learning_package_from_path(broken_fixture("missing_lp_key"), user=self.user) text = ctx.exception.as_text() assert text.startswith("Errors encountered during restore:\n") @@ -610,10 +610,10 @@ def test_wrapper_zip_matches_flat_archive(self): self.fixtures_folder, tmp_dir.name, prefix="MyLib/" ) - flat = api.create_learning_package( + flat = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib:flat:one" ) - wrapped = api.create_learning_package( + wrapped = api.load_learning_package_from_path( wrapped_zip, user=self.user, package_ref="lib:wrapped:zip" ) @@ -621,10 +621,10 @@ def test_wrapper_zip_matches_flat_archive(self): assert vars(wrapped.backup_metadata) == vars(flat.backup_metadata) def test_wrapper_directory_matches_flat_archive(self): - flat = api.create_learning_package( + flat = api.load_learning_package_from_path( self.fixtures_folder, user=self.user, package_ref="lib:flat:two" ) - wrapped = api.create_learning_package( + wrapped = api.load_learning_package_from_path( self.as_wrapper_dir(self.fixtures_folder), user=self.user, package_ref="lib:wrapped:dir", @@ -643,7 +643,7 @@ def test_macos_style_zip(self): extra_names=("__MACOSX/._MyLib", ".DS_Store"), ) - result = api.create_learning_package( + result = api.load_learning_package_from_path( mac_zip, user=self.user, package_ref="lib:wrapped:mac" ) @@ -665,7 +665,7 @@ def test_static_assets_survive_a_wrapper(self): self.fixtures_folder, tmp_dir.name, prefix="MyLib/" ) - result = api.create_learning_package( + result = api.load_learning_package_from_path( wrapped_zip, user=self.user, package_ref="lib:wrapped:static" ) @@ -693,7 +693,7 @@ def test_ambiguous_archive_is_refused(self): packages_before = publishing_api.LearningPackage.objects.count() with self.assertRaises(RestoreFailedError) as ctx: - api.create_learning_package( + api.load_learning_package_from_path( os.path.join(FIXTURES_DIR, "broken"), user=self.user ) @@ -709,7 +709,7 @@ def test_detected_root_is_reported_in_the_error_text(self): wrapper_dir = self.as_wrapper_dir(broken_fixture("missing_lp_key")) with self.assertRaises(RestoreFailedError) as ctx: - api.create_learning_package(wrapper_dir, user=self.user) + api.load_learning_package_from_path(wrapper_dir, user=self.user) text = ctx.exception.as_text() assert "Archive root: MyLib/" in text diff --git a/tests/openedx_content/applets/collections/test_api.py b/tests/openedx_content/applets/collections/test_api.py index 736de2f95..5838d7103 100644 --- a/tests/openedx_content/applets/collections/test_api.py +++ b/tests/openedx_content/applets/collections/test_api.py @@ -35,11 +35,11 @@ class CollectionTestCase(TestCase): @classmethod def setUpTestData(cls) -> None: - cls.learning_package = api.create_learning_package( + cls.learning_package = api.load_learning_package_from_path( package_ref="ComponentTestCase-test-key", title="Components Test Case Learning Package", ) - cls.learning_package_2 = api.create_learning_package( + cls.learning_package_2 = api.load_learning_package_from_path( package_ref="ComponentTestCase-test-key-2", title="Components Test Case another Learning Package", ) @@ -754,7 +754,7 @@ def test_set_collection_wrong_learning_package(self): """ We cannot set collections with a different learning package than the component. """ - learning_package_3 = api.create_learning_package( + learning_package_3 = api.load_learning_package_from_path( package_ref="ComponentTestCase-test-key-3", title="Components Test Case Learning Package-3", ) diff --git a/tests/openedx_content/applets/collections/test_signals.py b/tests/openedx_content/applets/collections/test_signals.py index 9b437fee2..f88b01b5a 100644 --- a/tests/openedx_content/applets/collections/test_signals.py +++ b/tests/openedx_content/applets/collections/test_signals.py @@ -19,7 +19,7 @@ @pytest.fixture(name="lp1") def _lp1() -> LearningPackage: """A learning package for use across collection signal tests.""" - return api.create_learning_package(package_ref="lp1", title="Test LP 📦") + return api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") def _create_entity(learning_package_id: LearningPackage.ID, entity_ref: str) -> PublishableEntity: diff --git a/tests/openedx_content/applets/publishing/test_signals.py b/tests/openedx_content/applets/publishing/test_signals.py index 405f933ab..a41ee69ad 100644 --- a/tests/openedx_content/applets/publishing/test_signals.py +++ b/tests/openedx_content/applets/publishing/test_signals.py @@ -44,7 +44,7 @@ def test_learning_package_created() -> None: is created. """ with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=1) as captured: - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") event = captured[0] assert event.signal is api.signals.LEARNING_PACKAGE_CREATED @@ -57,7 +57,7 @@ def test_learning_package_created_not_emitted_on_update() -> None: Test that updating an existing ``LearningPackage`` does NOT emit LEARNING_PACKAGE_CREATED. The event is only for new rows. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=0): api.update_learning_package(learning_package.id, title="Updated Title") @@ -70,7 +70,7 @@ def test_learning_package_created_aborted() -> None: """ with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=0): with abort_transaction(): - api.create_learning_package(package_ref="lp1", title="Test LP 📦") + api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") # LEARNING_PACKAGE_UPDATED @@ -82,7 +82,7 @@ def test_learning_package_updated() -> None: ``update_learning_package`` actually changes a field, and that the payload reflects the post-update title. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Original Title") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Original Title") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=1) as captured: api.update_learning_package(learning_package.id, title="New Title 📦") @@ -99,7 +99,7 @@ def test_learning_package_updated_noop() -> None: ``update_learning_package`` is called with no field changes (the early return in the API means the row is never saved). """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=0): api.update_learning_package(learning_package.id) @@ -110,7 +110,7 @@ def test_learning_package_updated_aborted() -> None: Test that LEARNING_PACKAGE_UPDATED is NOT emitted when the transaction that would have updated the ``LearningPackage`` is rolled back. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Original Title") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Original Title") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=0): with abort_transaction(): @@ -129,7 +129,7 @@ def test_learning_package_deleted() -> None: Test that LEARNING_PACKAGE_DELETED is emitted when a ``LearningPackage`` is deleted. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") lp_id = learning_package.id with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=1) as captured: @@ -146,8 +146,8 @@ def test_learning_package_deleted_via_queryset() -> None: Test that LEARNING_PACKAGE_DELETED fires once per row when multiple ``LearningPackage`` instances are deleted via a ``QuerySet.delete()``. """ - lp1 = api.create_learning_package(package_ref="lp1", title="LP 1") - lp2 = api.create_learning_package(package_ref="lp2", title="LP 2") + lp1 = api.load_learning_package_from_path(package_ref="lp1", title="LP 1") + lp2 = api.load_learning_package_from_path(package_ref="lp2", title="LP 2") with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=2) as captured: LearningPackage.objects.filter(id__in=[lp1.id, lp2.id]).delete() @@ -161,7 +161,7 @@ def test_learning_package_deleted_aborted() -> None: Test that LEARNING_PACKAGE_DELETED is NOT emitted when the transaction that would have deleted the ``LearningPackage`` is rolled back. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") lp_id = learning_package.id with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=0): @@ -182,7 +182,7 @@ def test_single_entity_changed() -> None: """ Test that ENTITIES_DRAFT_CHANGED is emitted when we change a publishable entity. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") # Note: creating an entity does not emit any events until we create a version of that entity. with capture_events(expected_count=0): @@ -220,7 +220,7 @@ def test_single_entity_changed_abort() -> None: Test that no events are emitted when we roll back a transaction that would have changed a publishable entity. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") entity = api.create_publishable_entity(learning_package.id, entity_ref="entity1", created=now_time, created_by=None) @@ -235,7 +235,7 @@ def test_multiple_entites_changed(admin_user) -> None: """ Test that ENTITIES_DRAFT_CHANGED is emitted when we change several publishable entities in a single edit. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Entity 1 will have no initial version: @@ -284,7 +284,7 @@ def test_multiple_entites_change_aborted() -> None: Test that ENTITIES_DRAFT_CHANGED is NOT emitted when we roll back a transaction that would have modified multiple entities in a bulk change. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Entity 1 will have no initial version: @@ -314,7 +314,7 @@ def test_changes_with_side_effects() -> None: Test that the ENTITIES_DRAFT_CHANGED event handles dependencies and side effects. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Create entities with dependencies @@ -349,7 +349,7 @@ def test_publish_events(admin_user) -> None: Test that ENTITIES_PUBLISHED is emitted when we publish changes to entities in a learning package. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Entity 1 will have no initial version: @@ -421,7 +421,7 @@ def test_publish_events_aborted(admin_user) -> None: Test that ENTITIES_PUBLISHED is NOT emitted when we roll back a transaction that would have published some entities. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Create an entity with some initial version: @@ -447,7 +447,7 @@ def test_publish_with_dependencies() -> None: Test that the ENTITIES_PUBLISHED event handles dependencies and side effects. """ - learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") + learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Create entities with dependencies From 72a30b573943374cb4d35a9ece0cd03ee539d661 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 29 Aug 2026 18:02:08 -0400 Subject: [PATCH 08/14] temp: remove bad side-effects from auto-renaming --- .../applets/backup_restore/payload.py | 25 ++++--------- .../applets/backup_restore/test_backup.py | 2 +- .../applets/collections/test_api.py | 6 ++-- .../applets/collections/test_signals.py | 2 +- .../applets/publishing/test_signals.py | 36 +++++++++---------- 5 files changed, 30 insertions(+), 41 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index bc759d9e9..6ef2674f7 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -29,12 +29,6 @@ ROOT_PACKAGE_PATH = "package.toml" -# Debris that archiving tools leave at the top level. These shouldn't count when -# we're working out whether an archive wraps its contents in a single folder. -# macOS's "Compress" in particular always adds __MACOSX, and often .DS_Store. -IGNORED_TOP_LEVEL_NAMES = frozenset({"__MACOSX"}) - - @attrs.define(frozen=True) class UnvalidatedLearningPackageInput: """ @@ -54,7 +48,8 @@ class UnvalidatedLearningPackageInput: # The folder inside the archive that we treated as the root, or None if the # archive's contents were at the top level. Every path in ``raw_data``, # ``entity_path_mapping`` and ``errors`` is relative to this, so it's only - # useful for telling a human what we decided. + # useful for telling a human what we decided. Note that ``fs`` itself is + # already relative to this root. root: str | None = None @@ -82,17 +77,11 @@ def find_archive_root( if fs.exists(root_package_path): return None - candidates = [] - # Note: this must be ls("") rather than ls("."), which returns [] on a - # ZipFileSystem -- i.e. exactly the case we're here to handle. - for entry in fs.ls("", detail=False): - name = entry.rsplit("/", 1)[-1] - if name in IGNORED_TOP_LEVEL_NAMES or name.startswith("."): - continue - if not fs.isdir(entry): - continue - if fs.exists(f"{entry}/{root_package_path}"): - candidates.append(entry) + candidates = [ + entry + for entry in fs.ls("/", detail=False) + if fs.isdir(entry) and fs.exists(f"{entry}/{root_package_path}") + ] # More than one candidate is ambiguous, and guessing would be worse than # saying we couldn't find the file. diff --git a/tests/openedx_content/applets/backup_restore/test_backup.py b/tests/openedx_content/applets/backup_restore/test_backup.py index aba0cabdb..1c9e5cf8b 100644 --- a/tests/openedx_content/applets/backup_restore/test_backup.py +++ b/tests/openedx_content/applets/backup_restore/test_backup.py @@ -50,7 +50,7 @@ def setUpTestData(cls): ) # Create a Learning Package for the test - cls.learning_package = api.load_learning_package_from_path( + cls.learning_package = api.create_learning_package( package_ref="ComponentTestCase-test-key", title="Components Test Case Learning Package", description="This is a test learning package for components.", diff --git a/tests/openedx_content/applets/collections/test_api.py b/tests/openedx_content/applets/collections/test_api.py index 5838d7103..736de2f95 100644 --- a/tests/openedx_content/applets/collections/test_api.py +++ b/tests/openedx_content/applets/collections/test_api.py @@ -35,11 +35,11 @@ class CollectionTestCase(TestCase): @classmethod def setUpTestData(cls) -> None: - cls.learning_package = api.load_learning_package_from_path( + cls.learning_package = api.create_learning_package( package_ref="ComponentTestCase-test-key", title="Components Test Case Learning Package", ) - cls.learning_package_2 = api.load_learning_package_from_path( + cls.learning_package_2 = api.create_learning_package( package_ref="ComponentTestCase-test-key-2", title="Components Test Case another Learning Package", ) @@ -754,7 +754,7 @@ def test_set_collection_wrong_learning_package(self): """ We cannot set collections with a different learning package than the component. """ - learning_package_3 = api.load_learning_package_from_path( + learning_package_3 = api.create_learning_package( package_ref="ComponentTestCase-test-key-3", title="Components Test Case Learning Package-3", ) diff --git a/tests/openedx_content/applets/collections/test_signals.py b/tests/openedx_content/applets/collections/test_signals.py index f88b01b5a..9b437fee2 100644 --- a/tests/openedx_content/applets/collections/test_signals.py +++ b/tests/openedx_content/applets/collections/test_signals.py @@ -19,7 +19,7 @@ @pytest.fixture(name="lp1") def _lp1() -> LearningPackage: """A learning package for use across collection signal tests.""" - return api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + return api.create_learning_package(package_ref="lp1", title="Test LP 📦") def _create_entity(learning_package_id: LearningPackage.ID, entity_ref: str) -> PublishableEntity: diff --git a/tests/openedx_content/applets/publishing/test_signals.py b/tests/openedx_content/applets/publishing/test_signals.py index a41ee69ad..405f933ab 100644 --- a/tests/openedx_content/applets/publishing/test_signals.py +++ b/tests/openedx_content/applets/publishing/test_signals.py @@ -44,7 +44,7 @@ def test_learning_package_created() -> None: is created. """ with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=1) as captured: - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") event = captured[0] assert event.signal is api.signals.LEARNING_PACKAGE_CREATED @@ -57,7 +57,7 @@ def test_learning_package_created_not_emitted_on_update() -> None: Test that updating an existing ``LearningPackage`` does NOT emit LEARNING_PACKAGE_CREATED. The event is only for new rows. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=0): api.update_learning_package(learning_package.id, title="Updated Title") @@ -70,7 +70,7 @@ def test_learning_package_created_aborted() -> None: """ with capture_events(signals=[api.signals.LEARNING_PACKAGE_CREATED], expected_count=0): with abort_transaction(): - api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + api.create_learning_package(package_ref="lp1", title="Test LP 📦") # LEARNING_PACKAGE_UPDATED @@ -82,7 +82,7 @@ def test_learning_package_updated() -> None: ``update_learning_package`` actually changes a field, and that the payload reflects the post-update title. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Original Title") + learning_package = api.create_learning_package(package_ref="lp1", title="Original Title") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=1) as captured: api.update_learning_package(learning_package.id, title="New Title 📦") @@ -99,7 +99,7 @@ def test_learning_package_updated_noop() -> None: ``update_learning_package`` is called with no field changes (the early return in the API means the row is never saved). """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=0): api.update_learning_package(learning_package.id) @@ -110,7 +110,7 @@ def test_learning_package_updated_aborted() -> None: Test that LEARNING_PACKAGE_UPDATED is NOT emitted when the transaction that would have updated the ``LearningPackage`` is rolled back. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Original Title") + learning_package = api.create_learning_package(package_ref="lp1", title="Original Title") with capture_events(signals=[api.signals.LEARNING_PACKAGE_UPDATED], expected_count=0): with abort_transaction(): @@ -129,7 +129,7 @@ def test_learning_package_deleted() -> None: Test that LEARNING_PACKAGE_DELETED is emitted when a ``LearningPackage`` is deleted. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") lp_id = learning_package.id with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=1) as captured: @@ -146,8 +146,8 @@ def test_learning_package_deleted_via_queryset() -> None: Test that LEARNING_PACKAGE_DELETED fires once per row when multiple ``LearningPackage`` instances are deleted via a ``QuerySet.delete()``. """ - lp1 = api.load_learning_package_from_path(package_ref="lp1", title="LP 1") - lp2 = api.load_learning_package_from_path(package_ref="lp2", title="LP 2") + lp1 = api.create_learning_package(package_ref="lp1", title="LP 1") + lp2 = api.create_learning_package(package_ref="lp2", title="LP 2") with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=2) as captured: LearningPackage.objects.filter(id__in=[lp1.id, lp2.id]).delete() @@ -161,7 +161,7 @@ def test_learning_package_deleted_aborted() -> None: Test that LEARNING_PACKAGE_DELETED is NOT emitted when the transaction that would have deleted the ``LearningPackage`` is rolled back. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") lp_id = learning_package.id with capture_events(signals=[api.signals.LEARNING_PACKAGE_DELETED], expected_count=0): @@ -182,7 +182,7 @@ def test_single_entity_changed() -> None: """ Test that ENTITIES_DRAFT_CHANGED is emitted when we change a publishable entity. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") # Note: creating an entity does not emit any events until we create a version of that entity. with capture_events(expected_count=0): @@ -220,7 +220,7 @@ def test_single_entity_changed_abort() -> None: Test that no events are emitted when we roll back a transaction that would have changed a publishable entity. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") entity = api.create_publishable_entity(learning_package.id, entity_ref="entity1", created=now_time, created_by=None) @@ -235,7 +235,7 @@ def test_multiple_entites_changed(admin_user) -> None: """ Test that ENTITIES_DRAFT_CHANGED is emitted when we change several publishable entities in a single edit. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Entity 1 will have no initial version: @@ -284,7 +284,7 @@ def test_multiple_entites_change_aborted() -> None: Test that ENTITIES_DRAFT_CHANGED is NOT emitted when we roll back a transaction that would have modified multiple entities in a bulk change. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Entity 1 will have no initial version: @@ -314,7 +314,7 @@ def test_changes_with_side_effects() -> None: Test that the ENTITIES_DRAFT_CHANGED event handles dependencies and side effects. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Create entities with dependencies @@ -349,7 +349,7 @@ def test_publish_events(admin_user) -> None: Test that ENTITIES_PUBLISHED is emitted when we publish changes to entities in a learning package. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Entity 1 will have no initial version: @@ -421,7 +421,7 @@ def test_publish_events_aborted(admin_user) -> None: Test that ENTITIES_PUBLISHED is NOT emitted when we roll back a transaction that would have published some entities. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args = {"created": now_time, "created_by": admin_user.id} # Create an entity with some initial version: @@ -447,7 +447,7 @@ def test_publish_with_dependencies() -> None: Test that the ENTITIES_PUBLISHED event handles dependencies and side effects. """ - learning_package = api.load_learning_package_from_path(package_ref="lp1", title="Test LP 📦") + learning_package = api.create_learning_package(package_ref="lp1", title="Test LP 📦") created_args: dict[str, Any] = {"created": now_time, "created_by": None} # Create entities with dependencies From 198aecf836ba9c7813ae60027abbb330ec0f686a Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 09:09:18 -0400 Subject: [PATCH 09/14] temp: move find_archive_root into PayloadExtractor Both find_archive_root and ROOT_PACKAGE_PATH are implementation details of this particular payload format, so they should be contained within this class. --- .../applets/backup_restore/loading.py | 7 +- .../applets/backup_restore/payload.py | 160 +++++++++--------- .../applets/backup_restore/validation.py | 5 +- .../applets/backup_restore/test_loading.py | 2 +- .../applets/backup_restore/test_payload.py | 42 +++-- 5 files changed, 111 insertions(+), 105 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py index 0382fb939..27ac2539d 100644 --- a/src/openedx_content/applets/backup_restore/loading.py +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -206,11 +206,12 @@ def _get_media_type(mime_type: str): media_type_str = media_type_str or "application/octet-stream" media_type = _get_media_type(media_type_str) - # TODO: Adopt data-urls for this. if path.startswith('static/'): + # TODO: data-url support # This is where we could add base64 encoded versions - # right now, we just use fs:/path/to/file - _resource_type, filepath = text_val.split(":", 1) + # when we introduce data-url support. But for now, just + # just assume these are file paths, e.g. path/to/file + filepath = text_val new_media = media_api.get_or_create_file_media( target.learning_package.id, media_type.id, diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index 6ef2674f7..77f63dc51 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -27,8 +27,6 @@ UnsupportedFormatError, ) -ROOT_PACKAGE_PATH = "package.toml" - @attrs.define(frozen=True) class UnvalidatedLearningPackageInput: """ @@ -53,44 +51,6 @@ class UnvalidatedLearningPackageInput: root: str | None = None -def find_archive_root( - fs: AbstractFileSystem, - root_package_path: str = ROOT_PACKAGE_PATH, -) -> str | None: - """ - Find the folder to treat as the archive root, or None to use ``fs`` as-is. - - People often build an archive by compressing a folder rather than that - folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a single - top-level directory with everything (including package.toml) inside it. That - is a reasonable thing to hand us, so we accept it. - - We only look one level down, and we require that the candidate directory - actually contains a ``root_package_path``. That second condition matters more - than it looks: without it, *any* archive whose top level happens to hold a - single directory would be re-rooted into it. - - This function never raises. An archive with no package.toml anywhere returns - None, and the missing file is reported later as an extraction error, which is - where that error belongs. - """ - if fs.exists(root_package_path): - return None - - candidates = [ - entry - for entry in fs.ls("/", detail=False) - if fs.isdir(entry) and fs.exists(f"{entry}/{root_package_path}") - ] - - # More than one candidate is ambiguous, and guessing would be worse than - # saying we couldn't find the file. - if len(candidates) == 1: - return candidates[0] - - return None - - class PayloadExtractor: """ Extracts files from a file system and generates unvalidated input. @@ -107,9 +67,11 @@ class PayloadExtractor: different set of conventions. For instance, the MIT Disciplinary Experts in Learning Technology and Applications team prefers to author in a way that encodes large parts of the hierarchy (Section -> Subsection -> Unit) in a - single file, with pointers to certain Components in different files. Such a - team would subclass this and override the handful of methods that know about - where files live and how they're shaped. + single file, with pointers to certain Components in different files. We are + not making this fully pluggable yet, but by keeping the payload extraction + separated from the archive container type (archive.py) on one side, and the + schema validation rules (validation.py) on the other side, we have the + flexbility to make the payload handling pluggable later on. Errors can happen at this layer, but they are errors related to the consistency of the archive payload format itself. So errors that need to be @@ -124,19 +86,62 @@ class PayloadExtractor: that are errors here are the things that prevent us from creating a UnvalidatedLearningPackageInput at all. """ + ROOT_PACKAGE_PATH = "package.toml" - # Declared as a class attribute so that a subclass using a different layout - # can point it somewhere else. - root_package_path = ROOT_PACKAGE_PATH + def __init__(self, source_fs: AbstractFileSystem): + """ + Initialize PayloadExtractor and auto-detect the root. - def __init__(self, fs: AbstractFileSystem): - self.source_fs = fs - self.root = find_archive_root(fs, self.root_package_path) + We expect the top level directory to have a ``package.toml`` file, an + ``entities`` directory, and a ``collections`` directory. A common error + when creating an archive is to zip up the parent directory instead. This + gives us a root that looks like:: - # Re-rooting with a DirFileSystem means that nothing below this line has - # to know whether the archive wrapped its contents in a folder: every - # path we read or report is relative to self.fs either way. - self.fs = DirFileSystem(path=self.root, fs=fs) if self.root else fs + some-folder-name/package.toml + some-folder-name/entities + some-folder-name/collections + + This is simple to check for and handle, so we just accept this kind of + input by wrapping the original source filesystem with a + ``DirFileSystem`` initialized with whatever the root path should be. + """ + self.root = self.find_archive_root(source_fs) + self.fs = DirFileSystem(path=self.root, fs=source_fs) if self.root else source_fs + + @classmethod + def find_archive_root(cls, source_fs: AbstractFileSystem) -> str | None: + """ + Find the folder to treat as the archive root, or None to use ``source_fs`` as-is. + + People often build an archive by compressing a folder rather than that + folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a single + top-level directory with everything (including package.toml) inside it. That + is a reasonable thing to hand us, so we accept it. + + We only look one level down, and we require that the candidate directory + actually contains a ``ROOT_PACKAGE_PATH``. That second condition matters more + than it looks: without it, *any* archive whose top level happens to hold a + single directory would be re-rooted into it. + + This function never raises. An archive with no package.toml anywhere returns + None, and the missing file is reported later as an extraction error, which is + where that error belongs. + """ + if source_fs.exists(cls.ROOT_PACKAGE_PATH): + return None + + candidates = [ + entry + for entry in source_fs.ls("/", detail=False) + if source_fs.isdir(entry) and source_fs.exists(f"{entry}/{cls.ROOT_PACKAGE_PATH}") + ] + + # More than one candidate is ambiguous, and guessing would be worse than + # saying we couldn't find the file. + if len(candidates) == 1: + return candidates[0] + + return None def extract(self) -> UnvalidatedLearningPackageInput: """ @@ -177,8 +182,6 @@ def extract(self) -> UnvalidatedLearningPackageInput: return UnvalidatedLearningPackageInput( raw_data=unvalidated, errors=errors, - # This must be the re-rooted filesystem, because the "fs:" static - # asset pointers we write below are relative to it. fs=self.fs, entity_path_mapping=entity_path_mapping, root=self.root, @@ -188,9 +191,9 @@ def extract_root_package_data(self, path: str | None = None) -> dict: """ Extract the "meta" and "learning_package" from the TOML file at path. - This is a straightforward extraction because we don't have to transform the - actual fields in the data. We expect to see a TOML file that looks something - like this: + This is a straightforward extraction because we don't have to transform + the actual fields in the data. We expect to see a TOML file that looks + something like this: [meta] format_version = 1 @@ -225,14 +228,18 @@ def extract_root_package_data(self, path: str | None = None) -> dict: } } - We need to return a Python dict that we get from parsing this. Most of this - method is error handling. The error checking at this layer is minimal, and - is mostly focused on making sure that the file exists, is parseable, and has - the two tables we expect it to have. + We need to return a Python dict that we get from parsing this. Most of + this method is error handling. The error checking at this layer is + minimal, and is mostly focused on making sure that the file exists, is + parseable, and has the two tables we expect it to have. + + Note that in any real world usage, we're just reading the default + ``ROOT_PACKAGE_PATH``. Passing the ``path`` explicitly just makes it + easier to have test package files with descriptive names for debugging. """ file_description = "Root Package" if path is None: - path = self.root_package_path + path = self.ROOT_PACKAGE_PATH # Check: Root Package file exists at all. if not self.fs.exists(path): @@ -271,16 +278,16 @@ def get_entity_file_paths(self) -> list[str]: """ Find all the PublishableEntity TOML file paths in our archive. - We expect our entity TOML files to be in the entities directory, but we have - two categories right now: + We expect our entity TOML files to be in the entities directory, but we + have two categories right now: * Component TOML: entities/xblock.v1/{component_type}/{component_code} * Container TOML: entities/{entity_ref} This method looks for TOML files in entities/ or any of its subdirs. We - only exclude matches inside the component_version data, to make sure that we - don't accidentally match media files in the unlikely event where people have - TOML files as static assets. + only exclude matches inside the component_version data, to make sure + that we don't accidentally match media files in the unlikely event where + people have TOML files as static assets. """ paths = [ path @@ -336,9 +343,9 @@ def extract_entity_data(self, path: str) -> tuple[str, dict]: """ This extracts raw entity data from an Entity TOML file. - PublishableEntities can be both Components (XBlock problems, videos, etc.), - as well as Containers like Units, Subsections, and Sections. Some sample - TOML: + PublishableEntities can be both Components (XBlock problems, videos, + etc.), as well as Containers like Units, Subsections, and Sections. Some + sample TOML: [entity] can_stand_alone = true @@ -410,10 +417,11 @@ def extract_entity_data(self, path: str) -> tuple[str, dict]: Note some key differences: 1. The "entity" table elements have been popped out to the top level. - 2. The "version" list has been renamed to "versions" to feel more natural. - 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back as - part of the tuple. This will become a key/value pair in an "entities" - dict that will hold all publishable entity input data. + 2. The "version" list has been renamed to "versions" to feel more + natural. + 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back + as part of the tuple. This will become a key/value pair in an + "entities" dict that will hold all publishable entity input data. """ file_description = "Entity" @@ -491,7 +499,7 @@ def _add_component_version_media(self, version: dict, entity_path: str) -> None: for static_file_path in self.fs.glob(f"{comp_ver_dir}/static/**"): if self.fs.isfile(static_file_path): rel_path = os.path.relpath(static_file_path, comp_ver_dir) - media[rel_path] = f"fs:{static_file_path}" + media[rel_path] = static_file_path version["component"] = {"media": media} diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py index 5423afa5f..c317033ea 100644 --- a/src/openedx_content/applets/backup_restore/validation.py +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -32,7 +32,7 @@ UnknownContainerTypeError, UnresolvedChildError, ) -from .payload import ROOT_PACKAGE_PATH, UnvalidatedLearningPackageInput +from .payload import PayloadExtractor, UnvalidatedLearningPackageInput from .schema import CompletePackageInputData @@ -123,7 +123,8 @@ def _source_for_loc( "location": tuple(rest), } case ("meta" | "learning_package", *_): - return {"path": ROOT_PACKAGE_PATH, "location": tuple(loc)} + # TODO: This is a problematic abstraction leak + return {"path": PayloadExtractor.ROOT_PACKAGE_PATH, "location": tuple(loc)} return {"path": None, "location": tuple(loc)} diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py index 1a0404b6d..b2e2e6702 100644 --- a/tests/openedx_content/applets/backup_restore/test_loading.py +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -654,7 +654,7 @@ def test_static_assets_survive_a_wrapper(self): """ Static files still resolve when the archive is wrapped. - The ``fs:`` pointers written during extraction are relative to the + The file pointers written during extraction are relative to the re-rooted filesystem, so this is what catches the case where the wrong filesystem gets handed downstream -- it would break images for wrapper archives only. diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py index 2f2084f8e..fd8d96eb8 100644 --- a/tests/openedx_content/applets/backup_restore/test_payload.py +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -6,10 +6,10 @@ This module tests our ability to extract data from the backup archive TOML files and resources, and assemble them into a combined document that represents the entire LearningPackage, and is encapsulated in UnvalidatedLearningPackageInput. -Most of these test methods that examine individual files. PayloadExtractor -takes the filesystem once, at construction, and its methods take a path, so it -should be possible to do simple test calls on TOML files and dirs without having -to mock anything. +Most of these test methods examine individual files. PayloadExtractor takes the +filesystem once, at construction, and its methods take a path, so it should be +possible to do simple test calls on TOML files and dirs without having to mock +anything. These tests are strictly for the payload module, and therefore don't need Django to run. @@ -233,14 +233,11 @@ def test_normal_component(self): assert versions_by_num[2]["component"]["media"] == { "block.xml": "

Version 2 text.

\n", } - # ...while static assets are encoded as "fs:" pointers back into the + # ...while static assets are encoded as pointers back into the # archive, so that we don't hold binary files in memory. v3_media = versions_by_num[3]["component"]["media"] assert v3_media["block.xml"] == "

Version 3 text.

\n" - assert v3_media["static/figure.png"].startswith("fs:") - assert v3_media["static/figure.png"].endswith( - "normal_component/component_versions/v3/static/figure.png" - ) + assert v3_media["static/figure.png"] == "normal_component/component_versions/v3/static/figure.png" def test_normal_container(self): ref, data = self.extractor.extract_entity_data("normal_container.toml") @@ -478,19 +475,19 @@ def setUp(self): def test_package_at_top_level_zip(self): fs = zip_fs_with(["package.toml", "entities/unit1.toml"]) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_package_at_top_level_dir(self): fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_single_wrapper_folder_zip(self): fs = zip_fs_with(["MyLib/package.toml", "MyLib/entities/unit1.toml"]) - assert payload.find_archive_root(fs) == "MyLib" + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" def test_single_wrapper_folder_dir(self): fs = dir_fs_with(self.tmp_path, {"MyLib/package.toml": self.PACKAGE}) - assert payload.find_archive_root(fs) == "MyLib" + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" def test_macos_style_zip(self): """ @@ -505,11 +502,11 @@ def test_macos_style_zip(self): "__MACOSX/._MyLib", ".DS_Store", ]) - assert payload.find_archive_root(fs) == "MyLib" + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" def test_wrapper_folder_beside_a_stray_file(self): fs = zip_fs_with(["MyLib/package.toml", "README.txt"]) - assert payload.find_archive_root(fs) == "MyLib" + assert payload.PayloadExtractor.find_archive_root(fs) == "MyLib" def test_folder_without_a_package_toml_is_not_a_root(self): """ @@ -519,20 +516,20 @@ def test_folder_without_a_package_toml_is_not_a_root(self): directory would be re-rooted into it. """ fs = zip_fs_with(["MyLib/entities/unit1.toml"]) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_two_candidate_folders_are_ambiguous(self): fs = zip_fs_with(["LibA/package.toml", "LibB/package.toml"]) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_nested_two_levels_is_not_followed(self): """We only look one level down; deeper nesting isn't worth guessing at.""" fs = zip_fs_with(["Outer/MyLib/package.toml"]) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_empty_archive(self): fs = dir_fs_with(self.tmp_path, {}) - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None def test_entities_fixture_is_not_re_rooted(self): """ @@ -543,7 +540,7 @@ def test_entities_fixture_is_not_re_rooted(self): silently re-root into it and break every entity test in this module. """ fs = DirFileSystem(TEST_DATA_ROOT / "entities") - assert payload.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) is None assert payload.PayloadExtractor(fs).root is None @@ -592,7 +589,7 @@ def test_paths_are_relative_to_the_detected_root(self): def test_static_asset_pointers_resolve_against_the_returned_fs(self): """ - The ``fs:`` pointers are relative to the *re-rooted* filesystem. + The file pointers are relative to the *re-rooted* filesystem. This is the subtle one: if extract() returned the original filesystem instead of the re-rooted one, these pointers would not resolve, and @@ -605,5 +602,4 @@ def test_static_asset_pointers_resolve_against_the_returned_fs(self): version = next(v for v in entity["versions"] if v["version_num"] == 5) pointer = version["component"]["media"]["static/me.png"] - assert pointer.startswith("fs:") - assert wrapped.fs.read_bytes(pointer.removeprefix("fs:")) + assert wrapped.fs.read_bytes(pointer) From f3f5756694c69fc51905e2ccfcd6eeca7ebd75c3 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 09:19:05 -0400 Subject: [PATCH 10/14] temp: reformat comment --- .../applets/backup_restore/payload.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index 77f63dc51..11d8fa814 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -114,18 +114,18 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str | None: Find the folder to treat as the archive root, or None to use ``source_fs`` as-is. People often build an archive by compressing a folder rather than that - folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a single - top-level directory with everything (including package.toml) inside it. That - is a reasonable thing to hand us, so we accept it. + folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a + single top-level directory with everything (including package.toml) + inside it. That is a reasonable thing to hand us, so we accept it. We only look one level down, and we require that the candidate directory - actually contains a ``ROOT_PACKAGE_PATH``. That second condition matters more - than it looks: without it, *any* archive whose top level happens to hold a - single directory would be re-rooted into it. + actually contains a ``ROOT_PACKAGE_PATH``. That second condition matters + more than it looks: without it, *any* archive whose top level happens to + hold a single directory would be re-rooted into it. - This function never raises. An archive with no package.toml anywhere returns - None, and the missing file is reported later as an extraction error, which is - where that error belongs. + This function never raises. An archive with no package.toml anywhere + returns None, and the missing file is reported later as an extraction + error, which is where that error belongs. """ if source_fs.exists(cls.ROOT_PACKAGE_PATH): return None From d00ab78d5e47910237bf4047a2b8a9e713317e82 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 09:46:37 -0400 Subject: [PATCH 11/14] refactor: check for duplicate entities during validation Entities and Collections were checked for duplicate keys in two different places, for a reason that was an accident of data structure rather than a decision: * Collections were assembled into a list, so both copies survived extraction and CompletePackageInputData.check_for_duplicate_keys caught the duplicate during validation, naming both files via each CollectionInput.src_path. * Entities were assembled into a dict keyed by entity ref, so a second definition would silently overwrite the first. PayloadExtractor therefore had to catch it during extraction, and UnvalidatedLearningPackageInput carried an entity_path_mapping side-channel so that later stages could work out which file an entity came from. Entities are now a list, each carrying its own key and src_path, and the duplicate check sits next to the Collections one, sharing a _reject_duplicate_keys helper. entity_path_mapping is gone, and _collection_path_at generalizes into _src_path_at(section, index, ...) now that both branches of _source_for_loc do the same thing. The list is the load-bearing part, and there is a comment in schema.py saying so: validation cannot report a duplicate that extraction has already destroyed, and a dict keyed by ref can only ever hold one of the two. Anyone "tidying" this back into a dict would silently remove the check. A missing entity key moves along with it. That was only checked during extraction because the key became a dict key and there was nowhere to put an entity without one; it is now a required field on EntityInputData, exactly as it is on CollectionInput. Deleted error classes --------------------- DuplicateFoundError and FieldMissing both lose their only caller here. Rather than leave two untested, uncovered exception classes behind, both are removed; they are easy to reintroduce if an extractor for some other archive layout turns out to need them. Path-less errors ---------------- A duplicate is reported against the whole section rather than one file -- pydantic gives a field_validator failure a loc of ("entities",), with no index to resolve to a path -- so SchemaError.path is None and the message names both files instead. Collections have always behaved this way. That surfaced a latent bug in the output: BackupRestoreError.__str__ interpolated self.path unconditionally, so any path-less error rendered as "None: entities: ..." in the restore log. Duplicate Collections have been producing that for as long as the check has existed; nothing asserted on it. Both __str__ methods now omit an absent path, and there are tests for it. Tests ----- 165 -> 175 passing in the applet, with payload.py, schema.py and errors.py staying at 100%. test_dupes inverts rather than moves: it now asserts that both duplicates *survive* extraction with distinct src_paths, which is precisely the property that lets validation do its job, so it earns its place as a payload-level test. test_missing_entity_key becomes a validation test. The helpers in test_validation take an explicit key and build lists, dropping the path_mapping parameter. No fixture files changed -- every archive fixture is still valid, only where the error surfaces has moved. Co-Authored-By: Claude Opus 5 (1M context) --- .../applets/backup_restore/errors.py | 36 ++-- .../applets/backup_restore/loading.py | 12 +- .../applets/backup_restore/payload.py | 67 +++---- .../applets/backup_restore/schema.py | 61 ++++-- .../applets/backup_restore/validation.py | 42 ++-- .../applets/backup_restore/test_loading.py | 23 ++- .../applets/backup_restore/test_payload.py | 123 +++++++----- .../applets/backup_restore/test_schema.py | 61 +++++- .../applets/backup_restore/test_validation.py | 189 +++++++++++++----- 9 files changed, 381 insertions(+), 233 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/errors.py b/src/openedx_content/applets/backup_restore/errors.py index 491ecb0bf..d7fb1baea 100644 --- a/src/openedx_content/applets/backup_restore/errors.py +++ b/src/openedx_content/applets/backup_restore/errors.py @@ -40,7 +40,12 @@ def __init__(self, message, path=None): self.path = path def __str__(self): - return f"{self.path}: {self.message}" + # Not every error is attributable to a file -- a duplicate key, for + # instance, is reported against the whole section rather than one path. + # Printing "None: ..." in a log file helps nobody. + if self.path: + return f"{self.path}: {self.message}" + return self.message class ArchiveNotReadableError(BackupRestoreError): @@ -80,19 +85,6 @@ def __init__(self, file_description, fields, path): super().__init__(message, path=path) -class FieldMissing(ExtractionError): - """A table is missing a field we need in order to go on.""" - - def __init__(self, file_description, table, missing_field, path): - self.table = table - self.missing_field = missing_field - message = ( - f'{file_description} is missing required field "{missing_field}" ' - f"from table [{table}]" - ) - super().__init__(message, path=path) - - class MissingFileError(ExtractionError): """ A file we require is not in the archive. @@ -107,13 +99,6 @@ def __init__(self, file_description, path): super().__init__(message, path=path) -class DuplicateFoundError(ExtractionError): - def __init__(self, description, original_path, path): - self.original_path = original_path - message = f"{description} already defined in {original_path}" - super().__init__(message, path=path) - - class UnsupportedFormatError(ExtractionError): """The archive declares a ``format_version`` we don't know how to read.""" @@ -141,10 +126,13 @@ def __init__(self, message, path=None, location=()): super().__init__(message, path=path) def __str__(self): + parts = [] + if self.path: + parts.append(str(self.path)) if self.location: - location_str = ".".join(str(part) for part in self.location) - return f"{self.path}: {location_str}: {self.message}" - return super().__str__() + parts.append(".".join(str(part) for part in self.location)) + parts.append(self.message) + return ": ".join(parts) class ConsistencyError(BackupRestoreError): diff --git a/src/openedx_content/applets/backup_restore/loading.py b/src/openedx_content/applets/backup_restore/loading.py index 27ac2539d..ccc1153e8 100644 --- a/src/openedx_content/applets/backup_restore/loading.py +++ b/src/openedx_content/applets/backup_restore/loading.py @@ -53,10 +53,14 @@ def __init__(self, validated_input: ValidatedLearningPackageInput): self.subsection_inputs: dict[str, EntityInputData] = {} self.unit_inputs: dict[str, EntityInputData] = {} - entities = self.data.entities + # Sorted so that creation order stays deterministic regardless of the + # order the archive's files happened to be read in. + entity_inputs = sorted(self.data.entities, key=lambda entity: entity.key) + self.entity_inputs_by_ref = {entity.key: entity for entity in entity_inputs} # Split our entities into separate dicts for convenience. - for entity_ref, entity_input in sorted(entities.items()): + for entity_input in entity_inputs: + entity_ref = entity_input.key match entity_input.container: case SectionInputData(): self.section_inputs[entity_ref] = entity_input @@ -329,14 +333,12 @@ def set_draft_versions(self, target: Target, for_publishing: bool): ``publish_all_drafts`` publishes the right thing. The second pass sets the drafts to their real values. """ - entity_inputs = self.data.entities - saved_entities = publishing_api.get_publishable_entities( target.learning_package.id ) for saved_entity in saved_entities: saved_draft_version = publishing_api.get_draft_version(saved_entity) - input_entity = entity_inputs[saved_entity.entity_ref] + input_entity = self.entity_inputs_by_ref[saved_entity.entity_ref] if for_publishing: input_version_num = input_entity.published.version_num diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index 11d8fa814..c95627260 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -17,9 +17,7 @@ from fsspec.implementations.dirfs import DirFileSystem from .errors import ( - DuplicateFoundError, ExtractionError, - FieldMissing, FieldsNotInTable, InvalidTOMLError, MissingFileError, @@ -27,6 +25,7 @@ UnsupportedFormatError, ) + @attrs.define(frozen=True) class UnvalidatedLearningPackageInput: """ @@ -40,14 +39,10 @@ class UnvalidatedLearningPackageInput: errors: list[ExtractionError] fs: AbstractFileSystem - # Mapping of entity refs to the paths where we found them. - entity_path_mapping: dict[str, str] - # The folder inside the archive that we treated as the root, or None if the - # archive's contents were at the top level. Every path in ``raw_data``, - # ``entity_path_mapping`` and ``errors`` is relative to this, so it's only - # useful for telling a human what we decided. Note that ``fs`` itself is - # already relative to this root. + # archive's contents were at the top level. Every path in ``raw_data`` and + # ``errors`` is relative to this, so it's only useful for telling a human + # what we decided. Note that ``fs`` itself is already relative to this root. root: str | None = None @@ -160,7 +155,7 @@ def extract(self) -> UnvalidatedLearningPackageInput: errors.append(err) # PublishableEntities & versions (components, units, sections, subsections) - entities_data, entity_path_mapping, entities_errors = self.extract_entities_data( + entities_data, entities_errors = self.extract_entities_data( self.get_entity_file_paths() ) unvalidated["entities"] = entities_data @@ -183,7 +178,6 @@ def extract(self) -> UnvalidatedLearningPackageInput: raw_data=unvalidated, errors=errors, fs=self.fs, - entity_path_mapping=entity_path_mapping, root=self.root, ) @@ -307,39 +301,26 @@ def extract_entities_data(self, paths: list[str]): """ Extract every entity file, collecting errors instead of raising them. - Returns a ``(entities_data, entity_path_mapping, errors)`` tuple. The - path mapping lets later stages report errors against the file an entity - came from, which is not derivable from the entity ref. + Returns an ``(entities_data, errors)`` tuple, where entities_data is a + list in the same order as ``paths``. - Duplicate detection lives here rather than in ``extract_entity_data`` - because it's a property of the *set* of files, not of any one file. + Note that we don't check for duplicate entity keys here. Each entity + records the file it came from, and they're kept in a list rather than + being collapsed into a dict, so a redefinition survives this stage + intact for validation to reject. That's the same division of labour we + use for Collections. """ - entities_data: dict[str, dict] = {} - entity_path_mapping: dict[str, str] = {} + entities_data: list[dict] = [] errors: list[ExtractionError] = [] for entity_file_path in paths: try: - entity_ref, entity_data = self.extract_entity_data(entity_file_path) - - # Check: Is it a duplicate of an Entity that has already been - # defined elsewhere in this archive? Without this, the second - # definition would silently overwrite the first, which would be - # baffling to someone assembling an archive by hand. - if entity_ref in entity_path_mapping: - raise DuplicateFoundError( - f"Entity key {entity_ref}", - entity_path_mapping[entity_ref], - entity_file_path, - ) - - entities_data[entity_ref] = entity_data - entity_path_mapping[entity_ref] = entity_file_path + entities_data.append(self.extract_entity_data(entity_file_path)) except ExtractionError as err: errors.append(err) - return entities_data, entity_path_mapping, errors + return entities_data, errors - def extract_entity_data(self, path: str) -> tuple[str, dict]: + def extract_entity_data(self, path: str) -> dict: """ This extracts raw entity data from an Entity TOML file. @@ -419,9 +400,8 @@ def extract_entity_data(self, path: str) -> tuple[str, dict]: 1. The "entity" table elements have been popped out to the top level. 2. The "version" list has been renamed to "versions" to feel more natural. - 3. The "key" field (a.k.a. entity_ref) has been popped out to pass back - as part of the tuple. This will become a key/value pair in an - "entities" dict that will hold all publishable entity input data. + 3. A "src_path" has been added, recording the file this came from, so + that validation can name it in an error message. """ file_description = "Entity" @@ -437,12 +417,11 @@ def extract_entity_data(self, path: str) -> tuple[str, dict]: if "entity" not in entity_root_dict: raise TableNotFoundError(file_description, "entity", path=path) - # Check: Does it define an Entity key (i.e. entity_ref)? We need to check - # this now because the dict we have to assemble will use these as keys. + # Note that we don't check for a missing "key" here. It's a required + # field on EntityInputData, so validation reports it, in the same way it + # does for a Collection with no key. entity = entity_root_dict["entity"] - entity_ref = entity.pop("key", None) - if not entity_ref: - raise FieldMissing(file_description, "entity", "key", path) + entity["src_path"] = path # Note case difference: we're renaming "version" in the TOML to "versions" # in the data dict we're assembling. @@ -450,7 +429,7 @@ def extract_entity_data(self, path: str) -> tuple[str, dict]: for version in entity["versions"]: self._add_component_version_media(version, path) - return entity_ref, entity + return entity def extract_collection_data(self, path: str) -> dict: """ diff --git a/src/openedx_content/applets/backup_restore/schema.py b/src/openedx_content/applets/backup_restore/schema.py index 030e710a1..25f3d50b1 100644 --- a/src/openedx_content/applets/backup_restore/schema.py +++ b/src/openedx_content/applets/backup_restore/schema.py @@ -9,7 +9,6 @@ """ from __future__ import annotations -from pathlib import Path from typing import Annotated, Literal from pydantic import ( @@ -43,6 +42,26 @@ ) +def _reject_duplicate_keys(items, label: str) -> None: + """ + Raise a ValueError if two items in ``items`` share a ``key``. + + Both Entities and Collections need this, and the message wants to name the + file that redefined the key as well as the one that got there first, which is + what each item's ``src_path`` is for. + """ + keys_to_items: dict[str, object] = {} + for item in items: + if item.key in keys_to_items: + original = keys_to_items[item.key] + raise ValueError( + f'{label} "{item.key}" redefined in ' + f'{item.src_path} (original in ' + f'{original.src_path})' # type: ignore[attr-defined] + ) + keys_to_items[item.key] = item + + class InputData(BaseModel): """ Base class for all inputs, here to set config defaults. @@ -69,14 +88,27 @@ class CompletePackageInputData(InputData): meta: MetaInputData learning_package: LearningPackageInputData - # Mapping of entity refs to EntityInputData - entities: dict[Annotated[str, REF_CONSTRAINTS], EntityInputData] + # These are lists rather than dicts keyed by their "key" field, and that is + # deliberate: a dict can only hold one entry per key, so a duplicated key + # would silently overwrite its predecessor during extraction and there would + # be nothing left for us to complain about here. Keeping them as lists is + # what lets the duplicate checks below exist at all. + entities: list[EntityInputData] collections: list[CollectionInput] + @field_validator('entities', mode='after') + @classmethod + def check_for_duplicate_entity_keys(cls, entities: list[EntityInputData]): + """ + Raise a ValueError if the same Entity is defined in two places. + """ + _reject_duplicate_keys(entities, "Entity") + return entities + @field_validator('collections', mode='after') @classmethod - def check_for_duplicate_keys(cls, collections: list[CollectionInput]): + def check_for_duplicate_collection_keys(cls, collections: list[CollectionInput]): """ Raise a ValueError if we encounter a duplicate collection entry. @@ -84,17 +116,7 @@ def check_for_duplicate_keys(cls, collections: list[CollectionInput]): entries (and other broken entries), while still otherwise allowing the restore to proceed. But for now, any error kills the restore process. """ - collection_keys_to_paths: dict[str, CollectionInput] = {} - for collection in collections: - if collection.key in collection_keys_to_paths: - originally_defined_collection = collection_keys_to_paths[collection.key] - raise ValueError( - f'Collection "{collection.key}" redefined in ' - f'{collection.src_path} (original in ' - f'{originally_defined_collection.src_path})' - ) - collection_keys_to_paths[collection.key] = collection - + _reject_duplicate_keys(collections, "Collection") return collections @@ -174,9 +196,10 @@ class PublishedInputData(InputData): class EntityInputData(InputData): """A PublishableEntity: either a Component or a Container.""" + key: Annotated[str, REF_CONSTRAINTS] + can_stand_alone: bool = True - # key: str created: AwareDatetime # Weird edge case: If you create something, never publish it, and then do a @@ -195,6 +218,10 @@ class EntityInputData(InputData): # TODO: Test unknown container type. container: UnitInputData | SubsectionInputData | SectionInputData | dict | None = None + # The source file this Entity was defined in. See the note on + # CollectionInput.src_path -- this is for error messages only. + src_path: str | None = None + class SectionInputData(InputData): """Marks an entity as a Section.""" @@ -262,7 +289,7 @@ class CollectionInput(InputData): # from this file directly because the exact format of this file should be # free to change as needed. That's the responsibilty of the payload.py # module. - src_path: Path | None = None + src_path: str | None = None # --- Output models. Not in use yet; the backup side still writes TOML directly. --- diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py index c317033ea..f62fda2dc 100644 --- a/src/openedx_content/applets/backup_restore/validation.py +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -74,7 +74,7 @@ def validate( errors.extend(_schema_errors_for(val_err, unvalidated_lp)) if data is not None: - errors.extend(_consistency_errors_for(data, unvalidated_lp)) + errors.extend(_consistency_errors_for(data)) return ValidatedLearningPackageInput( data=data, @@ -108,18 +108,14 @@ def _source_for_loc( Map a pydantic ``loc`` back to the archive file it came from. Pydantic reports errors against the combined document we assemble in - ``payload.py``, e.g. ``("entities", "unit1-b7eafb", "versions", 0, "title")``. - Nobody editing an archive has ever seen that document, so we split the ``loc`` - into the file it came from and the location within that file. + ``payload.py``, e.g. ``("entities", 3, "versions", 0, "title")``. Nobody + editing an archive has ever seen that document, so we split the ``loc`` into + the file it came from and the location within that file. """ match loc: - case ("entities", str() as entity_ref, *rest): - path = unvalidated_lp.entity_path_mapping.get(entity_ref) - # Fall back to naming the entity if we somehow have no path for it. - return {"path": path or f"entities/{entity_ref}", "location": tuple(rest)} - case ("collections", int() as index, *rest): + case ("entities" | "collections" as section, int() as index, *rest): return { - "path": _collection_path_at(index, unvalidated_lp), + "path": _src_path_at(section, index, unvalidated_lp), "location": tuple(rest), } case ("meta" | "learning_package", *_): @@ -129,27 +125,27 @@ def _source_for_loc( return {"path": None, "location": tuple(loc)} -def _collection_path_at( +def _src_path_at( + section: str, index: int, unvalidated_lp: UnvalidatedLearningPackageInput, ) -> str | None: """ - Look up the source file of the collection at ``index`` in the raw data. + Look up the source file of the item at ``index`` of ``section``. We can't read this off the validated model, because we only need it when validation has already failed. """ - raw_collections = unvalidated_lp.raw_data.get("collections", []) - if 0 <= index < len(raw_collections): - raw_collection = raw_collections[index] - if isinstance(raw_collection, dict): - return raw_collection.get("src_path") + raw_items = unvalidated_lp.raw_data.get(section, []) + if 0 <= index < len(raw_items): + raw_item = raw_items[index] + if isinstance(raw_item, dict): + return raw_item.get("src_path") return None def _consistency_errors_for( data: CompletePackageInputData, - unvalidated_lp: UnvalidatedLearningPackageInput, ) -> list[BackupRestoreError]: """ Check the cross-references that pydantic can't express. @@ -159,13 +155,11 @@ def _consistency_errors_for( the part of the archive that's actually wrong. """ errors: list[BackupRestoreError] = [] - known_refs = set(data.entities) - - def path_for(entity_ref: str) -> str | None: - return unvalidated_lp.entity_path_mapping.get(entity_ref) + known_refs = {entity.key for entity in data.entities} - for entity_ref, entity in sorted(data.entities.items()): - path = path_for(entity_ref) + for entity in sorted(data.entities, key=lambda e: e.key): + entity_ref = entity.key + path = entity.src_path # Check: is this a container type we actually know how to build? if isinstance(entity.container, dict): diff --git a/tests/openedx_content/applets/backup_restore/test_loading.py b/tests/openedx_content/applets/backup_restore/test_loading.py index b2e2e6702..8fc89deb2 100644 --- a/tests/openedx_content/applets/backup_restore/test_loading.py +++ b/tests/openedx_content/applets/backup_restore/test_loading.py @@ -28,7 +28,6 @@ from openedx_content.applets.backup_restore import api from openedx_content.applets.backup_restore.errors import ( ArchiveNotReadableError, - DuplicateFoundError, MissingFileError, RestoreFailedError, SchemaError, @@ -398,9 +397,17 @@ def test_unsupported_format_version(self): assert "2" in error.message def test_duplicate_entities(self): - error = self.assert_refuses("duplicate_entities", DuplicateFoundError) - assert error.path == "entities/second.toml" - assert error.original_path == "entities/first.toml" + """ + Two files declaring the same entity. Reported by validation, alongside + duplicate collections, rather than during extraction. + """ + error = self.assert_refuses("duplicate_entities", SchemaError) + + # A duplicate is reported against the section rather than one file, so + # the message names both, the same way a duplicate Collection does. + assert "entities/first.toml" in error.message + assert "entities/second.toml" in error.message + assert "unit1-b7eafb" in error.message def test_unknown_container_type(self): error = self.assert_refuses("unknown_container", UnknownContainerTypeError) @@ -561,11 +568,11 @@ def test_refuses_input_that_failed_validation(self): def test_refuses_an_unrecognized_container_type(self): data = CompletePackageInputData.model_construct( - entities={ - "chapter-1": EntityInputData.model_construct( - container={"chapter": {}}, versions=[] + entities=[ + EntityInputData.model_construct( + key="chapter-1", container={"chapter": {}}, versions=[] ) - } + ] ) with self.assertRaises(UnknownContainerTypeError): Loader(self._validated(data)) diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py index fd8d96eb8..4aee2e341 100644 --- a/tests/openedx_content/applets/backup_restore/test_payload.py +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -161,33 +161,46 @@ def test_missing_entity_table(self): assert "[entity]" in str(ctx.exception) def test_missing_entity_key(self): - with self.assertRaises(payload.FieldMissing) as ctx: - self.extractor.extract_entity_data("missing_entity_key.toml") - assert ctx.exception.missing_field == "key" - assert ctx.exception.table == "entity" + """ + An entity with no key extracts fine; rejecting it is validation's job. - def test_dupes(self): + See test_validation.DuplicateEntityTest.test_missing_entity_key. """ - Test for duplicate entities. + data = self.extractor.extract_entity_data("missing_entity_key.toml") + assert "key" not in data + assert data["src_path"] == "missing_entity_key.toml" - If we didn't explicitly check for this, a second file defining the same - entity one entity would just overwrite - the other, which would confuse people who might be assembling an archive - file for restoring. + def test_dupes(self): + """ + Two files declaring the same entity key both survive extraction. - This test is different from the others because extract_entities_data - doesn't raise exceptions, it collects them from its calls to - extract_entity_data(). + This is the property that lets validation reject the duplicate and name + both files. If we collapsed entities into a dict keyed by their ref, the + second definition would overwrite the first and there would be nothing + left to report. """ paths = ["dupe_1.toml", "dupe_2.toml"] - data, _path_mapping, errors = self.extractor.extract_entities_data(paths) - assert "dupe-key" in data # The first one should have succeeded... - assert len(data) == 1 # but the duplicate never made it in. - assert len(errors) == 1 # There should be only one error. + data, errors = self.extractor.extract_entities_data(paths) + + assert not errors + assert [entity["key"] for entity in data] == ["dupe-key", "dupe-key"] + # Each one knows where it came from, which is what makes the eventual + # error message actionable. + assert [entity["src_path"] for entity in data] == paths + + def test_broken_files_are_collected_not_raised(self): + """ + extract_entities_data gathers per-file errors instead of raising. + + One unreadable entity shouldn't stop us reporting on the rest of them. + """ + data, errors = self.extractor.extract_entities_data( + ["broken.toml", "normal_container.toml"] + ) - error = errors[0] - assert error.original_path == "dupe_1.toml" # path of the original - assert error.path == "dupe_2.toml" # path where error was marked + assert [type(err) for err in errors] == [payload.InvalidTOMLError] + assert errors[0].path == "broken.toml" + assert [entity["key"] for entity in data] == ["section-9-ac4b9f"] def test_ignore_unknown_tables(self): """ @@ -197,7 +210,7 @@ def test_ignore_unknown_tables(self): can add attributes without older code choking on them. Unknown tables at the top level are not part of the entity, so they don't come along. """ - _ref, data = self.extractor.extract_entity_data("unknown_table.toml") + data = self.extractor.extract_entity_data("unknown_table.toml") assert data["future_thing"] == {"some_setting": "hello"} assert "future_top_level" not in data @@ -208,14 +221,15 @@ def test_missing_versions(self): Whether that's actually loadable is the validation step's problem, not ours -- our job is only to faithfully report what's in the file. """ - ref, data = self.extractor.extract_entity_data("missing_versions.toml") - assert ref == "no-versions-c0ffee" + data = self.extractor.extract_entity_data("missing_versions.toml") + assert data["key"] == "no-versions-c0ffee" assert data["versions"] == [] assert data["container"] == {"unit": {}} def test_normal_component(self): - ref, data = self.extractor.extract_entity_data("normal_component.toml") - assert ref == "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" + data = self.extractor.extract_entity_data("normal_component.toml") + assert data["key"] == "xblock.v1:html:9f221fc4-42f1-4d07-ada4-653409bc5fff" + assert data["src_path"] == "normal_component.toml" assert data["can_stand_alone"] is True assert data["created"] == datetime( 2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc @@ -240,9 +254,10 @@ def test_normal_component(self): assert v3_media["static/figure.png"] == "normal_component/component_versions/v3/static/figure.png" def test_normal_container(self): - ref, data = self.extractor.extract_entity_data("normal_container.toml") - assert ref == "section-9-ac4b9f" + data = self.extractor.extract_entity_data("normal_container.toml") assert data == { + 'key': 'section-9-ac4b9f', + 'src_path': 'normal_container.toml', 'can_stand_alone': True, 'created': datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc), 'draft': { @@ -361,9 +376,9 @@ def test_normal(self): assert len(unvalidated.raw_data["entities"]) == 10 assert len(unvalidated.raw_data["collections"]) == 1 - def test_entity_path_mapping_uses_declared_key(self): + def test_each_entity_records_its_own_source_file(self): """ - The mapping is keyed by the entity's declared key, not its filename. + src_path is the file, key is what's declared inside it. Two fixture files deliberately have names that don't match the key inside them, because the export side hashes filenames to avoid @@ -372,11 +387,14 @@ def test_entity_path_mapping_uses_declared_key(self): fs = DirFileSystem(FIXTURES_ROOT / "library_backup") unvalidated = payload.PayloadExtractor(fs).extract() + paths_by_key = { + entity["key"]: entity["src_path"] + for entity in unvalidated.raw_data["entities"] + } assert ( - unvalidated.entity_path_mapping["section1-8ca126"] - == "entities/section1-extra-8ca126.toml" + paths_by_key["section1-8ca126"] == "entities/section1-extra-8ca126.toml" ) - assert unvalidated.entity_path_mapping[ + assert paths_by_key[ "xblock.v1:html:c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2" ].endswith("c22b9f97-f1e9-4e8f-87f0-d5a3c26083e2-extra.toml") @@ -390,22 +408,26 @@ def test_missing_root_package_is_collected_not_raised(self): assert error.path == "package.toml" # We still return a usable object, just an empty one. - assert unvalidated.raw_data["entities"] == {} + assert unvalidated.raw_data["entities"] == [] assert unvalidated.raw_data["collections"] == [] - def test_duplicate_entities_are_collected(self): + def test_duplicate_entities_both_survive_extraction(self): + """ + Extraction doesn't reject duplicates -- it preserves them for validation. + """ fs = DirFileSystem(TEST_DATA_ROOT / "duplicate_entities") unvalidated = payload.PayloadExtractor(fs).extract() - duplicate_errors = [ - err for err in unvalidated.errors - if isinstance(err, payload.DuplicateFoundError) + # This fixture is only the two entity files, so the absent package.toml + # is expected. What matters is that the duplicate isn't an error here. + assert [type(err) for err in unvalidated.errors] == [payload.MissingFileError] + + entities = unvalidated.raw_data["entities"] + assert [entity["key"] for entity in entities] == ["dupe-key", "dupe-key"] + assert [entity["src_path"] for entity in entities] == [ + "entities/dupe_1.toml", + "entities/dupe_2.toml", ] - assert len(duplicate_errors) == 1 - # The first file to declare the key wins; the second is the error. - assert duplicate_errors[0].original_path == "entities/dupe_1.toml" - assert duplicate_errors[0].path == "entities/dupe_2.toml" - assert "dupe-key" in unvalidated.raw_data["entities"] def test_static_assets_are_not_mistaken_for_entities(self): """ @@ -581,11 +603,11 @@ def test_raw_data_matches_the_flat_archive(self): def test_paths_are_relative_to_the_detected_root(self): wrapped = self.wrapped_extractor.extract() - assert wrapped.entity_path_mapping == self.flat.entity_path_mapping - assert ( - wrapped.entity_path_mapping["section1-8ca126"] - == "entities/section1-extra-8ca126.toml" - ) + paths_by_key = { + entity["key"]: entity["src_path"] + for entity in wrapped.raw_data["entities"] + } + assert paths_by_key["section1-8ca126"] == "entities/section1-extra-8ca126.toml" def test_static_asset_pointers_resolve_against_the_returned_fs(self): """ @@ -596,9 +618,10 @@ def test_static_asset_pointers_resolve_against_the_returned_fs(self): static assets would silently go missing for wrapper archives only. """ wrapped = self.wrapped_extractor.extract() - entity = wrapped.raw_data["entities"][ - "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" - ] + entity = next( + e for e in wrapped.raw_data["entities"] + if e["key"] == "xblock.v1:html:e32d5479-9492-41f6-9222-550a7346bc37" + ) version = next(v for v in entity["versions"] if v["version_num"] == 5) pointer = version["component"]["media"]["static/me.png"] diff --git a/tests/openedx_content/applets/backup_restore/test_schema.py b/tests/openedx_content/applets/backup_restore/test_schema.py index 92ada705d..a731c638e 100644 --- a/tests/openedx_content/applets/backup_restore/test_schema.py +++ b/tests/openedx_content/applets/backup_restore/test_schema.py @@ -33,7 +33,7 @@ def minimal_package(**overrides) -> dict: return { "meta": {"format_version": 1}, "learning_package": {"key": "lib:Axim:FunLib"}, - "entities": {}, + "entities": [], "collections": [], **overrides, } @@ -51,7 +51,7 @@ class ContainerDiscriminationTest(TestCase): """ def _container_for(self, raw): - entity = EntityInputData.model_validate({"created": CREATED, "container": raw}) + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED, "container": raw}) return entity.container def test_section(self): @@ -78,7 +78,7 @@ def test_no_container_means_component(self): assert self._container_for(None) is None def test_absent_container_defaults_to_none(self): - entity = EntityInputData.model_validate({"created": CREATED}) + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED}) assert entity.container is None @@ -179,7 +179,7 @@ class VersionInputTest(TestCase): def _entity_with_version(self, **version_overrides): version = {"version_num": 1, "title": "Some Title", **version_overrides} return EntityInputData.model_validate( - {"created": CREATED, "versions": [version]} + {"key": "some-entity", "created": CREATED, "versions": [version]} ) def test_blank_title_is_allowed(self): @@ -196,7 +196,7 @@ def test_version_num_must_be_positive(self): self._entity_with_version(version_num=0) def test_draft_and_published_default_to_none(self): - entity = EntityInputData.model_validate({"created": CREATED}) + entity = EntityInputData.model_validate({"key": "some-entity", "created": CREATED}) assert entity.draft.version_num is None assert entity.published.version_num is None @@ -206,7 +206,7 @@ def test_empty_published_table_means_unpublished(self): [entity.published] table rather than omitting it. """ entity = EntityInputData.model_validate( - {"created": CREATED, "published": {}, "draft": {"version_num": 2}} + {"key": "some-entity", "created": CREATED, "published": {}, "draft": {"version_num": 2}} ) assert entity.published.version_num is None assert entity.draft.version_num == 2 @@ -219,7 +219,7 @@ def test_minimal_package(self): data = CompletePackageInputData.model_validate(minimal_package()) assert data.learning_package.key == "lib:Axim:FunLib" - assert data.entities == {} + assert data.entities == [] assert data.collections == [] def test_duplicate_collection_keys_are_rejected(self): @@ -338,3 +338,50 @@ def test_a_genuinely_bad_email_is_still_rejected(self): MetaInputData.model_validate( {"format_version": 1, "created_by_email": "not-an-email"} ) + + +class DuplicateEntityKeyTest(TestCase): + """ + Entities are validated for duplicate keys the same way collections are. + + Both are declared as lists rather than dicts keyed by their key field, + precisely so that a redefinition survives extraction and can be reported + here instead of silently overwriting its predecessor. + """ + + def entity(self, key, src_path): + return {"key": key, "src_path": src_path, "created": CREATED} + + def test_duplicate_entity_keys_are_rejected(self): + raw = minimal_package(entities=[ + self.entity("unit1-b7eafb", "entities/first.toml"), + self.entity("unit1-b7eafb", "entities/second.toml"), + ]) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + message = str(ctx.exception) + assert "unit1-b7eafb" in message + assert "entities/first.toml" in message + assert "entities/second.toml" in message + + def test_distinct_entity_keys_are_fine(self): + raw = minimal_package(entities=[ + self.entity("unit1-b7eafb", "entities/first.toml"), + self.entity("unit2-c9dfa1", "entities/second.toml"), + ]) + data = CompletePackageInputData.model_validate(raw) + assert [e.key for e in data.entities] == ["unit1-b7eafb", "unit2-c9dfa1"] + + def test_key_is_required(self): + raw = minimal_package(entities=[{"created": CREATED}]) + with self.assertRaises(ValidationError) as ctx: + CompletePackageInputData.model_validate(raw) + + assert ("entities", 0, "key") in [err["loc"] for err in ctx.exception.errors()] + + def test_src_path_is_optional(self): + """It's only ever used to build error messages.""" + raw = minimal_package(entities=[{"key": "unit1-b7eafb", "created": CREATED}]) + data = CompletePackageInputData.model_validate(raw) + assert data.entities[0].src_path is None diff --git a/tests/openedx_content/applets/backup_restore/test_validation.py b/tests/openedx_content/applets/backup_restore/test_validation.py index 5f881b062..dfb9464aa 100644 --- a/tests/openedx_content/applets/backup_restore/test_validation.py +++ b/tests/openedx_content/applets/backup_restore/test_validation.py @@ -39,14 +39,27 @@ CREATED = datetime(2026, 4, 8, 15, 22, 12, 780012, tzinfo=timezone.utc) -def component(**overrides) -> dict: - """Raw data for a Component entity (i.e. one with no container).""" - return {"can_stand_alone": True, "created": CREATED, "versions": [], **overrides} +def component(key: str, **overrides) -> dict: + """ + Raw data for a Component entity (i.e. one with no container). + + Each entity carries its own key and source file, the same way collections + do, which is what lets validation report duplicates and attribute errors to + a file. + """ + return { + "key": key, + "src_path": f"entities/{key}.toml", + "can_stand_alone": True, + "created": CREATED, + "versions": [], + **overrides, + } -def container(kind: str, **overrides) -> dict: +def container(key: str, kind: str, **overrides) -> dict: """Raw data for a Container entity of the given kind.""" - return component(container={kind: {}}, **overrides) + return component(key, container={kind: {}}, **overrides) def version(version_num: int, children=None, **overrides) -> dict: @@ -56,14 +69,12 @@ def version(version_num: int, children=None, **overrides) -> dict: return raw -def unvalidated( - entities=None, collections=None, errors=None, path_mapping=None, root=None, **overrides -): +def unvalidated(entities=None, collections=None, errors=None, root=None, **overrides): """Build an UnvalidatedLearningPackageInput without going through files.""" raw_data = { "meta": {"format_version": 1}, "learning_package": {"key": "lib:Axim:FunLib"}, - "entities": entities if entities is not None else {}, + "entities": entities if entities is not None else [], "collections": collections if collections is not None else [], **overrides, } @@ -71,7 +82,6 @@ def unvalidated( raw_data=raw_data, errors=errors or [], fs=DirFileSystem(FIXTURES_ROOT), - entity_path_mapping=path_mapping or {}, root=root, ) @@ -93,12 +103,12 @@ def test_fs_is_passed_through(self): assert result.fs is source.fs def test_well_formed_container_tree(self): - result = validation.validate(unvalidated(entities={ - "section-1": container("section", versions=[version(1, ["subsection-1"])]), - "subsection-1": container("subsection", versions=[version(1, ["unit-1"])]), - "unit-1": container("unit", versions=[version(1, ["xblock.v1:html:abc"])]), - "xblock.v1:html:abc": component(versions=[version(1)]), - })) + result = validation.validate(unvalidated(entities=[ + container("section-1", "section", versions=[version(1, ["subsection-1"])]), + container("subsection-1", "subsection", versions=[version(1, ["unit-1"])]), + container("unit-1", "unit", versions=[version(1, ["xblock.v1:html:abc"])]), + component("xblock.v1:html:abc", versions=[version(1)]), + ])) assert result.errors == [] @@ -155,21 +165,24 @@ def test_meta_errors_point_at_package_toml(self): def test_entity_errors_point_at_the_entity_file(self): result = validation.validate(unvalidated( - entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, - path_mapping={"unit-1": "entities/unit1-b7eafb.toml"}, + entities=[container("unit1-b7eafb", "unit", versions=[{"version_num": 1}])], )) error = result.errors[0] assert error.path == "entities/unit1-b7eafb.toml" - # The entity ref is dropped, since the file only describes one entity. + # The list index is dropped, since the file only describes one entity. assert error.location == ("versions", 0, "title") def test_entity_errors_fall_back_when_path_is_unknown(self): + """An entity with no recorded source file still gets its error reported.""" result = validation.validate(unvalidated( - entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, + entities=[ + container("unit-1", "unit", versions=[{"version_num": 1}], src_path=None) + ], )) - assert result.errors[0].path == "entities/unit-1" + assert result.errors[0].path is None + assert result.errors[0].location == ("versions", 0, "title") def test_collection_errors_point_at_the_collection_file(self): result = validation.validate(unvalidated( @@ -182,8 +195,7 @@ def test_collection_errors_point_at_the_collection_file(self): def test_error_message_includes_location(self): result = validation.validate(unvalidated( - entities={"unit-1": container("unit", versions=[{"version_num": 1}])}, - path_mapping={"unit-1": "entities/unit1.toml"}, + entities=[container("unit1", "unit", versions=[{"version_num": 1}])], )) assert "entities/unit1.toml: versions.0.title" in str(result.errors[0]) @@ -197,8 +209,7 @@ class ConsistencyCheckTest(TestCase): def test_unresolved_child(self): result = validation.validate(unvalidated( - entities={"unit-1": container("unit", versions=[version(1, ["nope"])])}, - path_mapping={"unit-1": "entities/unit1.toml"}, + entities=[container("unit1", "unit", versions=[version(1, ["nope"])])], )) assert len(result.errors) == 1 @@ -208,20 +219,20 @@ def test_unresolved_child(self): assert "nope" in error.message def test_draft_pointing_at_a_missing_version(self): - result = validation.validate(unvalidated(entities={ - "unit-1": container("unit", draft={"version_num": 7}, versions=[version(1)]), - })) + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", draft={"version_num": 7}, versions=[version(1)]), + ])) assert len(result.errors) == 1 assert isinstance(result.errors[0], MissingVersionError) assert "[entity.draft]" in result.errors[0].message def test_published_pointing_at_a_missing_version(self): - result = validation.validate(unvalidated(entities={ - "unit-1": container( - "unit", published={"version_num": 7}, versions=[version(1)] + result = validation.validate(unvalidated(entities=[ + container( + "unit-1", "unit", published={"version_num": 7}, versions=[version(1)] ), - })) + ])) assert len(result.errors) == 1 assert isinstance(result.errors[0], MissingVersionError) @@ -229,15 +240,15 @@ def test_published_pointing_at_a_missing_version(self): def test_draft_and_published_may_be_absent(self): """An entity that was created and then reset to published has neither.""" - result = validation.validate(unvalidated(entities={ - "unit-1": container("unit", versions=[version(1)]), - })) + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", versions=[version(1)]), + ])) assert result.errors == [] def test_duplicate_version_num(self): - result = validation.validate(unvalidated(entities={ - "unit-1": container("unit", versions=[version(2), version(2)]), - })) + result = validation.validate(unvalidated(entities=[ + container("unit-1", "unit", versions=[version(2), version(2)]), + ])) duplicate_errors = [ err for err in result.errors if isinstance(err, DuplicateVersionError) @@ -250,24 +261,23 @@ def test_malformed_component_ref(self): Component refs are "{namespace}:{type}:{code}" -- the loader splits on the colons to work out what kind of block to build. """ - result = validation.validate(unvalidated(entities={ - "not-a-component-ref": component(versions=[version(1)]), - })) + result = validation.validate(unvalidated(entities=[ + component("not-a-component-ref", versions=[version(1)]), + ])) assert len(result.errors) == 1 assert isinstance(result.errors[0], MalformedRefError) def test_container_refs_are_not_required_to_have_colons(self): """Only Components derive meaning from the shape of their ref.""" - result = validation.validate(unvalidated(entities={ - "unit1-b7eafb": container("unit", versions=[version(1)]), - })) + result = validation.validate(unvalidated(entities=[ + container("unit1-b7eafb", "unit", versions=[version(1)]), + ])) assert result.errors == [] def test_unknown_container_type(self): result = validation.validate(unvalidated( - entities={"thing-1": component(container={"chapter": {}})}, - path_mapping={"thing-1": "entities/thing1.toml"}, + entities=[component("thing1", container={"chapter": {}})], )) assert len(result.errors) == 1 @@ -284,19 +294,19 @@ def test_consistency_checks_are_skipped_when_the_schema_is_broken(self): """ result = validation.validate(unvalidated( meta={}, - entities={"unit-1": container("unit", versions=[version(1, ["nope"])])}, + entities=[container("unit-1", "unit", versions=[version(1, ["nope"])])], )) assert result.data is None assert all(isinstance(err, SchemaError) for err in result.errors) def test_all_problems_are_reported_together(self): - result = validation.validate(unvalidated(entities={ - "unit-1": container( - "unit", draft={"version_num": 9}, versions=[version(1, ["nope"])] + result = validation.validate(unvalidated(entities=[ + container( + "unit-1", "unit", draft={"version_num": 9}, versions=[version(1, ["nope"])] ), - "bad-component-ref": component(versions=[version(1)]), - })) + component("bad-component-ref", versions=[version(1)]), + ])) error_types = {type(err) for err in result.errors} assert error_types == { @@ -338,13 +348,13 @@ class SourceMappingFallbackTest(TestCase): """ def test_unrecognized_location_has_no_path(self): - result = validation.validate(unvalidated(entities="not-a-dict")) + result = validation.validate(unvalidated(entities="not-a-list")) assert result.data is None error = result.errors[0] assert error.path is None assert error.location == ("entities",) - assert str(error).startswith("None: entities:") + assert str(error).startswith("entities:") def test_collection_without_a_src_path(self): result = validation.validate(unvalidated( @@ -389,3 +399,74 @@ def test_as_text_omits_the_root_when_there_isn_t_one(self): error = RestoreFailedError([MissingFileError("Root Package", path="package.toml")]) assert "Archive root" not in error.as_text() + + +class DuplicateEntityTest(TestCase): + """ + Two files defining the same entity key. + + This is checked here rather than during extraction for the same reason + duplicate Collections are: entities are kept in a list, so both definitions + survive extraction intact and validation can name both files. + """ + + def test_duplicate_entity_keys_are_rejected(self): + result = validation.validate(unvalidated(entities=[ + component("xblock.v1:html:abc", src_path="entities/first.toml"), + component("xblock.v1:html:abc", src_path="entities/second.toml"), + ])) + + assert result.data is None + assert len(result.errors) == 1 + error = result.errors[0] + assert isinstance(error, SchemaError) + # The message has to name both files to be actionable. + assert "entities/first.toml" in error.message + assert "entities/second.toml" in error.message + assert "xblock.v1:html:abc" in error.message + + def test_distinct_entity_keys_are_fine(self): + result = validation.validate(unvalidated(entities=[ + component("xblock.v1:html:abc"), + component("xblock.v1:html:def"), + ])) + + assert result.errors == [] + + def test_missing_entity_key(self): + """ + An entity file with no key at all. + + Extraction used to reject this, because the key became a dict key and it + had nowhere to put an entity without one. Now it's a required field like + any other. + """ + result = validation.validate(unvalidated( + entities=[{"created": CREATED, "src_path": "entities/no_key.toml"}], + )) + + assert result.data is None + error = result.errors[0] + assert isinstance(error, SchemaError) + assert error.path == "entities/no_key.toml" + assert error.location == ("key",) + + +class ErrorTextTest(TestCase): + """How individual errors render into the restore log.""" + + def test_an_error_with_no_path_omits_it(self): + """ + Not everything is attributable to a single file. + + A duplicate key is reported against the whole section, so there is no + one path to name -- and "None: ..." in a log file helps nobody. + """ + error = UnknownContainerTypeError("Entity declares an unsupported container") + + assert str(error) == "Entity declares an unsupported container" + + def test_an_error_with_a_path_names_it(self): + error = UnknownContainerTypeError("nope", path="entities/thing.toml") + + assert str(error) == "entities/thing.toml: nope" From 0c7de66f731b83410e1627c3a7d2d4a6469bfac5 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 18:00:54 -0400 Subject: [PATCH 12/14] refactor: let the filesystem carry the archive root PayloadExtractor wrapped its source filesystem in a DirFileSystem only when the archive turned out to have a wrapper folder: self.root = self.find_archive_root(source_fs) self.fs = DirFileSystem(path=self.root, fs=source_fs) if self.root else source_fs DirFileSystem already stores its root as .path, so in the wrapped case that value existed twice. And because the wrap was conditional, nothing downstream could rely on it being there, which is why the root was stored on PayloadExtractor, copied onto UnvalidatedLearningPackageInput, copied again onto ValidatedLearningPackageInput, and finally read once in api.py. Four hops to carry something the filesystem was already holding. We now wrap unconditionally. self.fs is a DirFileSystem in every case, fs.path is the archive-relative root, and all three stored copies are gone. find_archive_root returns "" rather than None for "no wrapper", so there is one spelling of that idea instead of two. Both input classes are typed `fs: DirFileSystem` rather than AbstractFileSystem. That narrowing is the point: it's what makes reading .path a stated invariant rather than an assumption about whatever happens to be there. Two things worth recording -------------------------- DirFileSystem(path="", ...) raises. Its __init__ does `path = path or fo`, so an empty string becomes None and _strip_protocol(None) fails with an AttributeError. "/" is the value that gives an identity wrap: both DirFileSystem and ZipFileSystem normalize it to "", after which _join and _relpath short-circuit and every path passes through untouched. There's a comment at the call site, because "/" reads as arbitrary otherwise. The local-directory case is now double-wrapped, deliberately. archive.read_fs_for_path already returns a DirFileSystem for a directory, rooted at an absolute local path. Reusing that directly would make fs.path mean two different things depending on how the archive arrived -- an absolute path for a directory, "MyLib" for a zip. The extra layer is what makes fs.path uniformly "the wrapper folder inside the archive, or empty". It costs nothing: with an empty root both hooks return their argument immediately. Tests ----- Mostly mechanical, `is None` -> `== ""`. The unvalidated() helper in test_validation now builds its filesystem the way PayloadExtractor does, so its root parameter still reads naturally at call sites. Added AlwaysRerootedTest, which is the guard for this whole change: it asserts the wrap happens for a flat zip *and* a flat directory, and checks that the source filesystem's own path is non-empty in the directory case. That last assertion is what fails if someone reintroduces the conditional wrap as an optimisation -- every other test in the suite would still pass while fs.path quietly went back to meaning two different things. Co-Authored-By: Claude Opus 5 (1M context) --- .../applets/backup_restore/api.py | 2 +- .../applets/backup_restore/payload.py | 39 ++++++---- .../applets/backup_restore/validation.py | 11 +-- .../applets/backup_restore/test_payload.py | 73 ++++++++++++++++--- .../applets/backup_restore/test_validation.py | 15 +++- 5 files changed, 104 insertions(+), 36 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/api.py b/src/openedx_content/applets/backup_restore/api.py index 5a6e4ac99..dd99352e5 100644 --- a/src/openedx_content/applets/backup_restore/api.py +++ b/src/openedx_content/applets/backup_restore/api.py @@ -64,7 +64,7 @@ def load_learning_package_from_path( # restore: a half-loaded Learning Package is harder to reason about than no # Learning Package at all. if validated_input.errors: - raise RestoreFailedError(validated_input.errors, validated_input.root) + raise RestoreFailedError(validated_input.errors, validated_input.fs.path) loader = loading.Loader(validated_input) archive_lp_input = loader.data.learning_package # LearningPackageInputData diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index c95627260..eda0d8a70 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -37,13 +37,13 @@ class UnvalidatedLearningPackageInput: raw_data: dict errors: list[ExtractionError] - fs: AbstractFileSystem - # The folder inside the archive that we treated as the root, or None if the - # archive's contents were at the top level. Every path in ``raw_data`` and - # ``errors`` is relative to this, so it's only useful for telling a human - # what we decided. Note that ``fs`` itself is already relative to this root. - root: str | None = None + # Always the re-rooted filesystem, never the one we were handed. ``fs.path`` + # is the folder inside the archive that we treated as the root, or "" if the + # contents were already at the top level -- so anyone who wants to tell a + # human what we decided can read it from here. Every path in ``raw_data`` and + # ``errors`` is relative to it. + fs: DirFileSystem class PayloadExtractor: @@ -99,14 +99,26 @@ def __init__(self, source_fs: AbstractFileSystem): This is simple to check for and handle, so we just accept this kind of input by wrapping the original source filesystem with a ``DirFileSystem`` initialized with whatever the root path should be. + + We wrap unconditionally, even when the archive is laid out correctly, so + that ``self.fs.path`` is a consistent answer to "what did we treat as the + root?" -- and so that we don't have to store that answer separately. """ - self.root = self.find_archive_root(source_fs) - self.fs = DirFileSystem(path=self.root, fs=source_fs) if self.root else source_fs + # "/" rather than "" because DirFileSystem's __init__ does + # `path = path or fo`, which turns an empty string into None. Both + # DirFileSystem and ZipFileSystem normalize "/" to "", which makes the + # wrapper a pass-through. + self.fs = DirFileSystem( + path=self.find_archive_root(source_fs) or "/", fs=source_fs + ) @classmethod - def find_archive_root(cls, source_fs: AbstractFileSystem) -> str | None: + def find_archive_root(cls, source_fs: AbstractFileSystem) -> str: """ - Find the folder to treat as the archive root, or None to use ``source_fs`` as-is. + Find the folder to treat as the archive root. + + Returns an empty string if the archive's contents are already at the top + level, i.e. there's no wrapper folder to descend into. People often build an archive by compressing a folder rather than that folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a @@ -119,11 +131,11 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str | None: hold a single directory would be re-rooted into it. This function never raises. An archive with no package.toml anywhere - returns None, and the missing file is reported later as an extraction + returns "", and the missing file is reported later as an extraction error, which is where that error belongs. """ if source_fs.exists(cls.ROOT_PACKAGE_PATH): - return None + return "" candidates = [ entry @@ -136,7 +148,7 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str | None: if len(candidates) == 1: return candidates[0] - return None + return "" def extract(self) -> UnvalidatedLearningPackageInput: """ @@ -178,7 +190,6 @@ def extract(self) -> UnvalidatedLearningPackageInput: raw_data=unvalidated, errors=errors, fs=self.fs, - root=self.root, ) def extract_root_package_data(self, path: str | None = None) -> dict: diff --git a/src/openedx_content/applets/backup_restore/validation.py b/src/openedx_content/applets/backup_restore/validation.py index f62fda2dc..d838fa1c9 100644 --- a/src/openedx_content/applets/backup_restore/validation.py +++ b/src/openedx_content/applets/backup_restore/validation.py @@ -20,7 +20,7 @@ from __future__ import annotations import attrs -from fsspec import AbstractFileSystem +from fsspec.implementations.dirfs import DirFileSystem from pydantic import ValidationError from .errors import ( @@ -49,14 +49,12 @@ class ValidatedLearningPackageInput: data: CompletePackageInputData | None - fs: AbstractFileSystem + # The re-rooted filesystem from extraction. ``fs.path`` is the folder inside + # the archive that was treated as its root, or "" if there wasn't one. + fs: DirFileSystem errors: list[BackupRestoreError] - # The folder inside the archive that was treated as its root, if any. Purely - # informational -- every path in ``errors`` is already relative to it. - root: str | None = None - def validate( unvalidated_lp: UnvalidatedLearningPackageInput, @@ -80,7 +78,6 @@ def validate( data=data, fs=unvalidated_lp.fs, errors=errors, - root=unvalidated_lp.root, ) diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py index 4aee2e341..25989d821 100644 --- a/tests/openedx_content/applets/backup_restore/test_payload.py +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -497,11 +497,11 @@ def setUp(self): def test_package_at_top_level_zip(self): fs = zip_fs_with(["package.toml", "entities/unit1.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_package_at_top_level_dir(self): fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_single_wrapper_folder_zip(self): fs = zip_fs_with(["MyLib/package.toml", "MyLib/entities/unit1.toml"]) @@ -538,20 +538,20 @@ def test_folder_without_a_package_toml_is_not_a_root(self): directory would be re-rooted into it. """ fs = zip_fs_with(["MyLib/entities/unit1.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_two_candidate_folders_are_ambiguous(self): fs = zip_fs_with(["LibA/package.toml", "LibB/package.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_nested_two_levels_is_not_followed(self): """We only look one level down; deeper nesting isn't worth guessing at.""" fs = zip_fs_with(["Outer/MyLib/package.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_empty_archive(self): fs = dir_fs_with(self.tmp_path, {}) - assert payload.PayloadExtractor.find_archive_root(fs) is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" def test_entities_fixture_is_not_re_rooted(self): """ @@ -562,8 +562,61 @@ def test_entities_fixture_is_not_re_rooted(self): silently re-root into it and break every entity test in this module. """ fs = DirFileSystem(TEST_DATA_ROOT / "entities") - assert payload.PayloadExtractor.find_archive_root(fs) is None - assert payload.PayloadExtractor(fs).root is None + assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor(fs).fs.path == "" + + +class AlwaysRerootedTest(TestCase): + """ + PayloadExtractor always wraps its source filesystem, wrapper or not. + + This is what lets ``fs.path`` be the single answer to "what did we treat as + the root?", which is why nothing stores that separately any more. Skipping + the wrap when no wrapper folder is found would pass every other test in this + module and quietly leave ``fs.path`` meaning two different things: "" for a + zip, but an absolute local path for a directory. + """ + + PACKAGE = "[meta]\nformat_version = 1\n" + + def setUp(self): + super().setUp() + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp_path = self._tmp.name + + def test_flat_zip_is_still_wrapped(self): + extractor = payload.PayloadExtractor(zip_fs_with(["package.toml"])) + + assert isinstance(extractor.fs, DirFileSystem) + assert extractor.fs.path == "" + + def test_flat_directory_is_still_wrapped(self): + source_fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) + extractor = payload.PayloadExtractor(source_fs) + + assert isinstance(extractor.fs, DirFileSystem) + # Not the absolute local path that source_fs is rooted at. + assert extractor.fs.path == "" + assert source_fs.path != "" + + def test_wrapped_archive_records_the_folder(self): + extractor = payload.PayloadExtractor(zip_fs_with(["MyLib/package.toml"])) + + assert isinstance(extractor.fs, DirFileSystem) + assert extractor.fs.path == "MyLib" + + def test_the_wrapper_is_a_pass_through_when_there_is_no_root(self): + """An identity wrap must not disturb any path operation.""" + source_fs = dir_fs_with(self.tmp_path, { + "package.toml": self.PACKAGE, + "entities/unit1.toml": "[entity]\nkey = \"unit1\"\n", + }) + extractor = payload.PayloadExtractor(source_fs) + + assert extractor.fs.exists("package.toml") + assert extractor.fs.glob("entities/*.toml") == ["entities/unit1.toml"] + assert extractor.fs.read_text("package.toml") == self.PACKAGE class ExtractThroughWrapperTest(TestCase): @@ -585,14 +638,14 @@ def setUpClass(cls): cls.wrapped_extractor = payload.PayloadExtractor(wrapped_fs) def test_flat_archive_has_no_root(self): - assert self.flat.root is None + assert self.flat.fs.path == "" def test_wrapper_root_is_detected(self): """ ``fixtures/`` holds library_backup (with a package.toml) and broken/ (without one), so library_backup is the only candidate. """ - assert self.wrapped_extractor.root == "library_backup" + assert self.wrapped_extractor.fs.path == "library_backup" def test_raw_data_matches_the_flat_archive(self): wrapped = self.wrapped_extractor.extract() diff --git a/tests/openedx_content/applets/backup_restore/test_validation.py b/tests/openedx_content/applets/backup_restore/test_validation.py index dfb9464aa..07319798b 100644 --- a/tests/openedx_content/applets/backup_restore/test_validation.py +++ b/tests/openedx_content/applets/backup_restore/test_validation.py @@ -81,8 +81,9 @@ def unvalidated(entities=None, collections=None, errors=None, root=None, **overr return UnvalidatedLearningPackageInput( raw_data=raw_data, errors=errors or [], - fs=DirFileSystem(FIXTURES_ROOT), - root=root, + # Built the way PayloadExtractor builds it: always a DirFileSystem, whose + # .path is the wrapper folder inside the archive (or "" if there is none). + fs=DirFileSystem(path=root or "/", fs=DirFileSystem(FIXTURES_ROOT)), ) @@ -378,10 +379,16 @@ class ArchiveRootPassthroughTest(TestCase): """ def test_root_is_carried_through(self): - assert validation.validate(unvalidated(root="MyLib")).root == "MyLib" + """ + The root travels on the filesystem itself, not as a separate field. + + DirFileSystem already records what it was rooted at, so there is nothing + for validation to copy across. + """ + assert validation.validate(unvalidated(root="MyLib")).fs.path == "MyLib" def test_no_root_by_default(self): - assert validation.validate(unvalidated()).root is None + assert validation.validate(unvalidated()).fs.path == "" def test_as_text_names_the_root_when_there_is_one(self): error = RestoreFailedError( From cc41f3b49b43c762173f359dc2528e03233d28a5 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 20:42:25 -0400 Subject: [PATCH 13/14] temp: minor comment addition --- .../applets/backup_restore/payload.py | 76 +++++++++++-------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/payload.py b/src/openedx_content/applets/backup_restore/payload.py index eda0d8a70..eb2cf8923 100644 --- a/src/openedx_content/applets/backup_restore/payload.py +++ b/src/openedx_content/applets/backup_restore/payload.py @@ -31,18 +31,18 @@ class UnvalidatedLearningPackageInput: """ Everything we could pull out of an archive, before validation. - ``raw_data`` is the assembled document we hand to pydantic. + ``raw_data`` is the assembled document we hand to pydantic validation (into + the ``CompletePackageInputData`` model). ``errors`` holds anything that stopped us assembling part of it. """ - raw_data: dict errors: list[ExtractionError] # Always the re-rooted filesystem, never the one we were handed. ``fs.path`` # is the folder inside the archive that we treated as the root, or "" if the # contents were already at the top level -- so anyone who wants to tell a - # human what we decided can read it from here. Every path in ``raw_data`` and - # ``errors`` is relative to it. + # human what we decided can read it from here. Every path in ``raw_data`` + # and ``errors`` is relative to this root. fs: DirFileSystem @@ -101,15 +101,17 @@ def __init__(self, source_fs: AbstractFileSystem): ``DirFileSystem`` initialized with whatever the root path should be. We wrap unconditionally, even when the archive is laid out correctly, so - that ``self.fs.path`` is a consistent answer to "what did we treat as the - root?" -- and so that we don't have to store that answer separately. + that ``self.fs.path`` is a consistent answer to "what did we treat as + the root?" -- and so that we don't have to store that answer separately. + This still works even if ``source_fs`` is another ``DirFileSystem``. """ - # "/" rather than "" because DirFileSystem's __init__ does - # `path = path or fo`, which turns an empty string into None. Both - # DirFileSystem and ZipFileSystem normalize "/" to "", which makes the - # wrapper a pass-through. + # Note: We can't pass path="" because of how DirFileSystem is + # implemented (it will become ``None``, and that will raise errors). + # Instead, we pass path="/" when things are where they should be at the + # top level. Confusingly, DirFileSystem will normalize this to "", so + # if you later query for it, expect self.fs.path == "" for this case. self.fs = DirFileSystem( - path=self.find_archive_root(source_fs) or "/", fs=source_fs + path=self.find_archive_root(source_fs), fs=source_fs ) @classmethod @@ -117,8 +119,8 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str: """ Find the folder to treat as the archive root. - Returns an empty string if the archive's contents are already at the top - level, i.e. there's no wrapper folder to descend into. + Returns "/" if the archive's contents are already at the top level, i.e. + there's no wrapper folder to descend into. People often build an archive by compressing a folder rather than that folder's contents, e.g. ``zip -r MyLib.zip MyLib/``. The result has a @@ -126,16 +128,16 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str: inside it. That is a reasonable thing to hand us, so we accept it. We only look one level down, and we require that the candidate directory - actually contains a ``ROOT_PACKAGE_PATH``. That second condition matters - more than it looks: without it, *any* archive whose top level happens to - hold a single directory would be re-rooted into it. + actually contains a ``ROOT_PACKAGE_PATH``. This function never raises. An archive with no package.toml anywhere - returns "", and the missing file is reported later as an extraction - error, which is where that error belongs. + returns "/", and the missing file is reported later as an extraction + error, which is where that error belongs. Note that both DirFileSystem + and ZipFileSystem normalize "/" to "", which makes the wrapper a + pass-through. """ if source_fs.exists(cls.ROOT_PACKAGE_PATH): - return "" + return "/" candidates = [ entry @@ -148,14 +150,23 @@ def find_archive_root(cls, source_fs: AbstractFileSystem) -> str: if len(candidates) == 1: return candidates[0] - return "" + # If we get here, the archive is broken, but we return the top-level + # anyway and let other error-handling code flag that package.toml is + # missing. + return "/" def extract(self) -> UnvalidatedLearningPackageInput: """ Read the whole archive, gathering errors rather than raising them. + + The general philosophy here is to always march on and get as much as + possible, even if we know the upload is doomed. It's better to see all + the errors for an archive at once, than to see only one error and have + to go through the upload cycle to see the next issue. """ - # The general philosophy here is to always march on and get as much as - # possible, even if we know the upload is doomed. + # The ``unvalidated`` dict will eventually be used to load a + # ``CompletePackageInputData``, so it needs to match its fields: + # ``meta``, ``learning_package``, ``entities``, and ``collections``. unvalidated: dict = {} errors: list[ExtractionError] = [] @@ -263,10 +274,10 @@ def extract_root_package_data(self, path: str | None = None) -> dict: file_description, table="learning_package", path=path ) - # Check: We only support format_version 1, and don't know what to do with - # anything higher. This leaves us some wiggle-room to declare a 1.x version - # that is backwards compatible, i.e. it will reject 2 and higher, but accept - # 1.1, 1.2, etc. + # Check: We only support format_version 1, and don't know what to do + # with anything higher. This leaves us some wiggle-room to declare a 1.x + # version that is backwards compatible, i.e. it will reject 2 and + # higher, but accept 1.1, 1.2, etc. format_version = root_package_dict["meta"].get("format_version") is_number = isinstance(format_version, (int, float)) and not isinstance( format_version, bool @@ -422,9 +433,10 @@ def extract_entity_data(self, path: str) -> dict: self._check_all_fields_in_tables(entity_root_dict, file_description, path) # Check: Does it define a top level "[entity]" table? Note that this can - # pass if they define a sub-table like "[entity.draft]", since the existence - # of "[entity]" is implicit in that case. If we get that far, rely on - # catching it at the validation step (i.e. after payload extraction). + # pass if they define a sub-table like "[entity.draft]", since the + # existence of "[entity]" is implicit in that case. If we get that far, + # rely on catching it at the validation step (i.e. after payload + # extraction). if "entity" not in entity_root_dict: raise TableNotFoundError(file_description, "entity", path=path) @@ -434,8 +446,8 @@ def extract_entity_data(self, path: str) -> dict: entity = entity_root_dict["entity"] entity["src_path"] = path - # Note case difference: we're renaming "version" in the TOML to "versions" - # in the data dict we're assembling. + # Note case difference: we're renaming "version" in the TOML to + # "versions" in the data dict we're assembling. entity["versions"] = entity_root_dict.pop("version", []) for version in entity["versions"]: self._add_component_version_media(version, path) @@ -485,7 +497,7 @@ def _add_component_version_media(self, version: dict, entity_path: str) -> None: if self.fs.isfile(media_path) } # Any static files are encoded as pointers. - # TODO: Convert this to data-urls later + # TODO: Add support for data-urls for static_file_path in self.fs.glob(f"{comp_ver_dir}/static/**"): if self.fs.isfile(static_file_path): rel_path = os.path.relpath(static_file_path, comp_ver_dir) From e56cc9c70497f81d8739c2797fc69304bdaac4ab Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 30 Aug 2026 21:00:13 -0400 Subject: [PATCH 14/14] temp: fix tests related to re-rooting --- .../applets/backup_restore/schema.py | 47 ------------------- .../applets/backup_restore/test_payload.py | 14 +++--- 2 files changed, 7 insertions(+), 54 deletions(-) diff --git a/src/openedx_content/applets/backup_restore/schema.py b/src/openedx_content/applets/backup_restore/schema.py index 25f3d50b1..23d02bb5e 100644 --- a/src/openedx_content/applets/backup_restore/schema.py +++ b/src/openedx_content/applets/backup_restore/schema.py @@ -290,50 +290,3 @@ class CollectionInput(InputData): # free to change as needed. That's the responsibilty of the payload.py # module. src_path: str | None = None - - -# --- Output models. Not in use yet; the backup side still writes TOML directly. --- - - -class PackageConfigOutputData(BaseModel): - """ - Writes the package.toml file when we're writing a backup archive. - """ - meta: MetaOutputData - learning_package: LearningPackageOutputData - - -class MetaOutputData(BaseModel): - """ - Output Package Metadata - - This is metadata that is written so that people can more easily figure out - where a backup archive came from. - - The "created_by", "created_by_email", and "created_at" fields all refer to - the user who created the backup archive, not the user who created the - Library (Learning Package). - """ - format_version: Literal[1] - created_by: StrictStr = Field(min_length=1) - created_by_email: EmailStr - created_at: AwareDatetime - origin_server: StrictStr - - -class LearningPackageOutputData(BaseModel): - """ - High level data for a Learning Package. - """ - title: StrictStr = Field(min_length=1) - key: StrictStr = Field( - pattern=r"^lib:[\w\-.]+:[\w\-.]+$", - description="This is a LibraryLocatorV2", - examples=[ - "lib:OrgName:LibraryName", - "lib:Axim:IntroPhysics", - ] - ) - description: StrictStr - created: AwareDatetime - updated: AwareDatetime diff --git a/tests/openedx_content/applets/backup_restore/test_payload.py b/tests/openedx_content/applets/backup_restore/test_payload.py index 25989d821..d3821ee6c 100644 --- a/tests/openedx_content/applets/backup_restore/test_payload.py +++ b/tests/openedx_content/applets/backup_restore/test_payload.py @@ -497,11 +497,11 @@ def setUp(self): def test_package_at_top_level_zip(self): fs = zip_fs_with(["package.toml", "entities/unit1.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_package_at_top_level_dir(self): fs = dir_fs_with(self.tmp_path, {"package.toml": self.PACKAGE}) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_single_wrapper_folder_zip(self): fs = zip_fs_with(["MyLib/package.toml", "MyLib/entities/unit1.toml"]) @@ -538,20 +538,20 @@ def test_folder_without_a_package_toml_is_not_a_root(self): directory would be re-rooted into it. """ fs = zip_fs_with(["MyLib/entities/unit1.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_two_candidate_folders_are_ambiguous(self): fs = zip_fs_with(["LibA/package.toml", "LibB/package.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_nested_two_levels_is_not_followed(self): """We only look one level down; deeper nesting isn't worth guessing at.""" fs = zip_fs_with(["Outer/MyLib/package.toml"]) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_empty_archive(self): fs = dir_fs_with(self.tmp_path, {}) - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" def test_entities_fixture_is_not_re_rooted(self): """ @@ -562,7 +562,7 @@ def test_entities_fixture_is_not_re_rooted(self): silently re-root into it and break every entity test in this module. """ fs = DirFileSystem(TEST_DATA_ROOT / "entities") - assert payload.PayloadExtractor.find_archive_root(fs) == "" + assert payload.PayloadExtractor.find_archive_root(fs) == "/" assert payload.PayloadExtractor(fs).fs.path == ""