[WIP] Re-implementing Restore to use Pydantic (attempt 2)

docs/openedx_content/decisions/0025-backup-restore.rst

@@ -0,0 +1,46 @@
+25. Learning Package Serialization and Validation Approach
+==========================================================
+
+Context
+-------
+
+Content Libraries map 1:1 to LearningPackages and these need to be imported and exported as file archives. Initial support for this was released in Ulmo, but we wanted to revisit it to make it more robust during the Verawood timeline. This is part of that effort.
+
+* Flexibility of Structure
+* Standardization of validation (JSON Schema)
+* Justify ZIP
+* Justify TOML
+* Max 100,000 items.
+* Use of fsspec as abstraction
+
+Phases
+
+Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage
+
+
+Decision
+--------
+
+Some key points:
+
+1. We intentionally separate input and output formats because the output format
+   will change over time, but the various input formats must continue to be
+   supported. We don't inherit from one from the other because we don't *want*
+   those changes to be automatically propogated--that breaks compatibility.
+2. We assemble into giant JSON in order to simplify validation and allow for
+   more flexibility in structural representation. There's the archive layer and
+   then the logical layer and then serialization into the database.
+
+
+Archive -> Model (validation) + Resources -> Database
+
+
+
+Consequences
+------------
+
+
+
+Rejected alternatives
+---------------------
+

mypy.ini

@@ -12,5 +12,8 @@ files =
 [mypy-organizations.*]
 follow_untyped_imports = True
 
+[mypy-fsspec.*]
+follow_untyped_imports = True
+
 [mypy.plugins.django-stubs]
 django_settings_module = "projects.dev"

requirements/base.in

@@ -17,3 +17,7 @@ rules<4.0                 # Django extension for rules-based authorization check
 tomlkit                   # Parses and writes TOML configuration files
 
 edx-organizations         # Implemented the "Organization" model that CatalogCourse/CourseRun are keyed to
+
+fsspec                    # Used by openedx_content's backup_restore to abstract zip access
+
+pydantic[email]           # Used by openedx_content's backup_restore for input validation

requirements/base.txt

@@ -6,6 +6,8 @@
 #
 amqp==5.3.1
     # via kombu
+annotated-types==0.8.0
+    # via pydantic
 asgiref==3.12.1
     # via django
 attrs==26.1.0
@@ -24,7 +26,7 @@ cffi==2.1.1
     #   pynacl
 charset-normalizer==3.5.1
     # via requests
-click==8.4.2
+click==8.5.0
     # via
     #   celery
     #   click-didyoumean
@@ -37,7 +39,7 @@ click-plugins==1.1.1.2
     # via celery
 click-repl==0.3.0
     # via celery
-cryptography==50.0.0
+cryptography==50.0.1
     # via pyjwt
 django==5.2.17
     # via
@@ -70,7 +72,9 @@ djangorestframework==3.18.0
     #   edx-drf-extensions
     #   edx-organizations
 dnspython==2.8.0
-    # via pymongo
+    # via
+    #   email-validator
+    #   pymongo
 drf-jwt==1.19.2
     # via edx-drf-extensions
 edx-ccx-keys==2.0.2
@@ -91,10 +95,16 @@ edx-opaque-keys[django]==4.0.0
     #   openedx-events
 edx-organizations==9.0.0
     # via -r requirements/base.in
+email-validator==2.3.0
+    # via pydantic
 fastavro==1.12.2
     # via openedx-events
+fsspec==2026.7.0
+    # via -r requirements/base.in
 idna==3.19
-    # via requests
+    # via
+    #   email-validator
+    #   requests
 kombu==5.6.2
     # via celery
 openedx-events==11.2.0
@@ -109,6 +119,10 @@ psutil==7.2.2
     # via edx-django-utils
 pycparser==3.0
     # via cffi
+pydantic[email]==2.13.5
+    # via -r requirements/base.in
+pydantic-core==2.46.5
+    # via pydantic
 pyjwt[crypto]==2.13.0
     # via
     #   drf-jwt
@@ -138,7 +152,13 @@ stevedore==5.9.1
 tomlkit==0.15.1
     # via -r requirements/base.in
 typing-extensions==4.16.0
-    # via edx-opaque-keys
+    # via
+    #   edx-opaque-keys
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.4
+    # via pydantic
 tzdata==2026.3
     # via kombu
 tzlocal==5.4.4
@@ -150,7 +170,7 @@ vine==5.1.0
     #   amqp
     #   celery
     #   kombu
-wcwidth==0.8.2
+wcwidth==0.8.3
     # via prompt-toolkit
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/ci.txt

@@ -19,23 +19,23 @@ packaging==26.3
     # via
     #   pyproject-api
     #   tox
-platformdirs==4.11.3
+platformdirs==4.11.5
     # via
     #   tox
     #   virtualenv
 pluggy==1.6.0
     # via tox
 pyproject-api==1.11.0
     # via tox
-python-discovery==1.5.2
+python-discovery==1.6.0
     # via
     #   tox
     #   virtualenv
 tomli-w==1.2.0
     # via tox
-tox==4.60.0
+tox==4.60.1
     # via -r requirements/ci.in
 typing-extensions==4.16.0
     # via tox
-virtualenv==21.7.4
+virtualenv==21.7.7
     # via tox

requirements/dev.txt

@@ -8,6 +8,10 @@ amqp==5.3.1
     # via
     #   -r requirements/quality.txt
     #   kombu
+annotated-types==0.8.0
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 asgiref==3.12.1
     # via
     #   -r requirements/quality.txt
@@ -29,7 +33,7 @@ billiard==4.2.4
     # via
     #   -r requirements/quality.txt
     #   celery
-build==1.5.0
+build==1.6.0
     # via
     #   -r requirements/pip-tools.txt
     #   pip-tools
@@ -54,7 +58,7 @@ charset-normalizer==3.5.1
     # via
     #   -r requirements/quality.txt
     #   requests
-click==8.4.2
+click==8.5.0
     # via
     #   -r requirements/pip-tools.txt
     #   -r requirements/quality.txt
@@ -96,7 +100,7 @@ coverage[toml]==7.15.4
     # via
     #   -r requirements/quality.txt
     #   pytest-cov
-cryptography==50.0.0
+cryptography==50.0.1
     # via
     #   -r requirements/quality.txt
     #   pyjwt
@@ -166,11 +170,12 @@ djangorestframework==3.18.0
     #   drf-jwt
     #   edx-drf-extensions
     #   edx-organizations
-djangorestframework-stubs==3.18.0
+djangorestframework-stubs==3.18.1
     # via -r requirements/quality.txt
 dnspython==2.8.0
     # via
     #   -r requirements/quality.txt
+    #   email-validator
     #   pymongo
 docutils==0.23
     # via
@@ -206,6 +211,10 @@ edx-opaque-keys[django]==4.0.0
     #   openedx-events
 edx-organizations==9.0.0
     # via -r requirements/quality.txt
+email-validator==2.3.0
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 fastavro==1.12.2
     # via
     #   -r requirements/quality.txt
@@ -218,7 +227,9 @@ filelock==3.32.4
     #   virtualenv
 freezegun==1.5.5
     # via -r requirements/quality.txt
-grimp==3.15
+fsspec==2026.7.0
+    # via -r requirements/quality.txt
+grimp==3.16
     # via
     #   -r requirements/quality.txt
     #   import-linter
@@ -229,8 +240,9 @@ id==1.6.1
 idna==3.19
     # via
     #   -r requirements/quality.txt
+    #   email-validator
     #   requests
-import-linter==2.13
+import-linter==2.14
     # via -r requirements/quality.txt
 iniconfig==2.3.0
     # via
@@ -341,7 +353,7 @@ pillow==12.3.0
     #   edx-organizations
 pip-tools==7.6.1
     # via -r requirements/pip-tools.txt
-platformdirs==4.11.3
+platformdirs==4.11.5
     # via
     #   -r requirements/ci.txt
     #   -r requirements/quality.txt
@@ -372,6 +384,12 @@ pycparser==3.0
     # via
     #   -r requirements/quality.txt
     #   cffi
+pydantic[email]==2.13.5
+    # via -r requirements/quality.txt
+pydantic-core==2.46.5
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 pydocstyle==6.3.0
     # via -r requirements/quality.txt
 pygments==2.21.0
@@ -437,7 +455,7 @@ python-dateutil==2.9.0.post0
     #   -r requirements/quality.txt
     #   celery
     #   freezegun
-python-discovery==1.5.2
+python-discovery==1.6.0
     # via
     #   -r requirements/ci.txt
     #   tox
@@ -451,7 +469,7 @@ pyyaml==6.0.3
     #   -r requirements/quality.txt
     #   code-annotations
     #   edx-i18n-tools
-readme-renderer==45.0
+readme-renderer==46.0
     # via
     #   -r requirements/quality.txt
     #   twine
@@ -518,7 +536,7 @@ tomlkit==0.15.1
     #   -r requirements/quality.txt
     #   edx-lint
     #   pylint
-tox==4.60.0
+tox==4.60.1
     # via -r requirements/ci.txt
 twine==7.0.0
     # via -r requirements/quality.txt
@@ -537,7 +555,14 @@ typing-extensions==4.16.0
     #   edx-opaque-keys
     #   import-linter
     #   mypy
+    #   pydantic
+    #   pydantic-core
     #   tox
+    #   typing-inspection
+typing-inspection==0.4.4
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 tzdata==2026.3
     # via
     #   -r requirements/quality.txt
@@ -558,11 +583,11 @@ vine==5.1.0
     #   amqp
     #   celery
     #   kombu
-virtualenv==21.7.4
+virtualenv==21.7.7
     # via
     #   -r requirements/ci.txt
     #   tox
-wcwidth==0.8.2
+wcwidth==0.8.3
     # via
     #   -r requirements/quality.txt
     #   prompt-toolkit