From 12edcdc43c8a93e49f698618793a6b7fc8268aa0 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Mon, 26 Oct 2026 13:44:05 +0000 Subject: [PATCH 1/7] Add missing django migration When forking from WebODM the defaults were changed. This doesn't affect the database but does avoid them being bundled in a later change. I also added a CI check to catch this kind of missing migration. --- .github/workflows/test-docker.yml | 1 + ...lter_setting_organization_name_and_more.py | 79 +++++++++++++++++++ app/models/README.md | 19 +++++ 3 files changed, 99 insertions(+) create mode 100644 app/migrations/0054_alter_setting_organization_name_and_more.py create mode 100644 app/models/README.md diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 32325bbd..2e51e4f8 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -29,6 +29,7 @@ jobs: docker compose -f docker-compose.yml -f docker-compose.build.yml build --build-arg TEST_BUILD=ON docker compose -f docker-compose.yml -f docker-compose.build.yml up -d sleep 20 + docker compose exec -T webapp python manage.py makemigrations --check --dry-run docker compose exec -T webapp /webui/webui.sh test image: diff --git a/app/migrations/0054_alter_setting_organization_name_and_more.py b/app/migrations/0054_alter_setting_organization_name_and_more.py new file mode 100644 index 00000000..6f0e6ae1 --- /dev/null +++ b/app/migrations/0054_alter_setting_organization_name_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 6.0.7 on 2026-10-26 11:56 + +import colorfield.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0053_alter_basemap_id_alter_plugindatum_id_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='setting', + name='organization_name', + field=models.CharField(blank=True, default='OpenDroneMap', help_text='The name of your organization', max_length=255, null=True, verbose_name='Organization name'), + ), + migrations.AlterField( + model_name='setting', + name='organization_website', + field=models.URLField(blank=True, default='https://opendronemap.org', help_text='The website URL of your organization', max_length=255, null=True, verbose_name='Organization website'), + ), + migrations.AlterField( + model_name='theme', + name='button_danger', + field=colorfield.fields.ColorField(default='#cb4b16', help_text='Delete button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Danger'), + ), + migrations.AlterField( + model_name='theme', + name='button_default', + field=colorfield.fields.ColorField(default='#60818e', help_text='Default button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Default'), + ), + migrations.AlterField( + model_name='theme', + name='button_primary', + field=colorfield.fields.ColorField(default='#07485b', help_text='Primary button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Primary'), + ), + migrations.AlterField( + model_name='theme', + name='dialog_warning', + field=colorfield.fields.ColorField(default='#cb4b16', help_text='The border color of warning dialogs.', image_field=None, max_length=25, samples=None, verbose_name='Dialog Warning'), + ), + migrations.AlterField( + model_name='theme', + name='header_background', + field=colorfield.fields.ColorField(default='#60818e', help_text="Background color of the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Background'), + ), + migrations.AlterField( + model_name='theme', + name='header_primary', + field=colorfield.fields.ColorField(default='#ecf9fb', help_text="Text and icons in the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Primary'), + ), + migrations.AlterField( + model_name='theme', + name='highlight', + field=colorfield.fields.ColorField(default='#eee8d5', help_text='The background color of panels and some borders.', image_field=None, max_length=25, samples=None, verbose_name='Highlight'), + ), + migrations.AlterField( + model_name='theme', + name='primary', + field=colorfield.fields.ColorField(default='#07485b', help_text='Most text, icons, and borders.', image_field=None, max_length=25, samples=None, verbose_name='Primary'), + ), + migrations.AlterField( + model_name='theme', + name='secondary', + field=colorfield.fields.ColorField(default='#fef5e6', help_text='The main background color, and text color of some buttons.', image_field=None, max_length=25, samples=None, verbose_name='Secondary'), + ), + migrations.AlterField( + model_name='theme', + name='success', + field=colorfield.fields.ColorField(default='#cbd198', help_text='The background color of success notifications.', image_field=None, max_length=25, samples=None, verbose_name='Success'), + ), + migrations.AlterField( + model_name='theme', + name='tertiary', + field=colorfield.fields.ColorField(default='#0d6986', help_text='Navigation links.', image_field=None, max_length=25, samples=None, verbose_name='Tertiary'), + ), + ] diff --git a/app/models/README.md b/app/models/README.md new file mode 100644 index 00000000..8d31e4a8 --- /dev/null +++ b/app/models/README.md @@ -0,0 +1,19 @@ +# Models and migrations + +The classes in this directory and in `nodeodm/models.py` define the database tables. + +After changing a model generate the matching migration and commit it with the change: + +``` +docker compose exec webapp python manage.py makemigrations +``` + +The command writes a new numbered file under the app's `migrations/` directory. + +Migrations are applied automatically by `start.sh` when the webapp container starts. Nothing needs to run `migrate` by hand. + +Two branches that each add a migration with the same parent produce the error `Conflicting migrations detected; multiple leaf nodes`. Resolve it with: + +``` +docker compose exec webapp python manage.py makemigrations --merge +``` From 739270104790aaa43bb37c2efde91df97100e822 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Mon, 26 Oct 2026 14:18:25 +0000 Subject: [PATCH 2/7] Use historical models in nodeodm data migrations 0008 and 0010 imported ProcessingNode from nodeodm.models, so their queries selected every column the current model has. Adding a field to ProcessingNode in a later migration would make fresh installs fail at 0008 with "column does not exist". Resolve the model through apps.get_model instead, as the Django migrations docs require. --- nodeodm/migrations/0008_rename_default_odm_node.py | 2 +- nodeodm/migrations/0010_rename_odm_to_odx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nodeodm/migrations/0008_rename_default_odm_node.py b/nodeodm/migrations/0008_rename_default_odm_node.py index b66f4f1c..aa153252 100644 --- a/nodeodm/migrations/0008_rename_default_odm_node.py +++ b/nodeodm/migrations/0008_rename_default_odm_node.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals from django.db import migrations -from nodeodm.models import ProcessingNode def rename_default_node(apps, schema_editor): + ProcessingNode = apps.get_model('nodeodm', 'ProcessingNode') for default_node in ProcessingNode.objects.filter(hostname='node-odm-1'): default_node.hostname = 'webui_node-odm_1' default_node.label = 'node-odm-1' diff --git a/nodeodm/migrations/0010_rename_odm_to_odx.py b/nodeodm/migrations/0010_rename_odm_to_odx.py index d00d9a38..78bd052b 100644 --- a/nodeodm/migrations/0010_rename_odm_to_odx.py +++ b/nodeodm/migrations/0010_rename_odm_to_odx.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from django.db import migrations -from nodeodm.models import ProcessingNode from django.db.models import Q def rename_default_node(apps, schema_editor): + ProcessingNode = apps.get_model('nodeodm', 'ProcessingNode') for default_node in ProcessingNode.objects.filter(Q(hostname='node-odm-1') | Q(label='node-odm-1')): default_node.hostname = 'node-odm-1' default_node.label = 'node-odm-1' From 46387cc18cd6f651750ab0533d1e740f93581544 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Tue, 27 Oct 2026 14:27:15 +0000 Subject: [PATCH 3/7] Fully migrate db from postgres 9.5 to 18 There are two main variants we need to consider. 1. A default user will have the data stored in a docker volume. 2. Expert users may use --db-dir to store the data on the native filesystem. Both cases have extra checks in place to ensure we do not lose any data. The full migration in both cases is carefully constructed to allow WebODM to function afterwards if the migration fails. Data will always be preserved. On first boot of WebUI, we check to see if the WebUI database exists. If it doesn't, we prompt for the migration process. If accepted, we start up both the old and new database and run `pg_dump` and pipe it to the new database. Finally once we know the data is secure, in the case of a native disk, we swap the new and old files. This gives us an updated Postgres database. We don't copy or modify the media files to avoid excessive disk consumption. The next step is to migrate the Django models. This is done automatically when the app is first run. If the user declines to migrate their data, it is possible to still do the migration as long as the new database remains empty. The implementation is split into a python file `importwebodm.py` and a bash file `import-webodm.sh`. This is to aid reusability. --- app/management/commands/importwebodm.py | 136 ++++++++++++++++ app/tests/test_importwebodm.py | 64 ++++++++ docker-compose.yml | 2 + import-webodm.sh | 206 ++++++++++++++++++++++++ webui.sh | 12 ++ 5 files changed, 420 insertions(+) create mode 100644 app/management/commands/importwebodm.py create mode 100644 app/tests/test_importwebodm.py create mode 100644 import-webodm.sh diff --git a/app/management/commands/importwebodm.py b/app/management/commands/importwebodm.py new file mode 100644 index 00000000..ad83de94 --- /dev/null +++ b/app/management/commands/importwebodm.py @@ -0,0 +1,136 @@ +import os +import subprocess +import time + +from django.core.management.base import BaseCommand, CommandError +from django.db import connection + +DEFAULT_NODE_NAME = 'node-odm-1' +LEGACY_NODE_NAME = 'node-odx-1' +ANONYMOUS_USER = 'AnonymousUser' + + +def pg_args(params): + args = [] + if params.get('host'): + args += ['-h', params['host']] + if params.get('port'): + args += ['-p', str(params['port'])] + if params.get('user'): + args += ['-U', params['user']] + return args + + +def pg_env(params): + env = os.environ.copy() + if params.get('password'): + env['PGPASSWORD'] = params['password'] + return env + + +def psql(params, sql, dbname=None): + result = subprocess.run(['psql'] + pg_args(params) + ['-d', dbname or params['dbname'], '-tAc', sql], + env=pg_env(params), capture_output=True, text=True) + if result.returncode != 0: + raise CommandError("psql failed: {}".format(result.stderr.strip())) + return result.stdout.strip() + + +def wait_for_database(params, timeout=120): + deadline = time.monotonic() + timeout + while True: + try: + psql(params, 'SELECT 1') + return + except CommandError: + if time.monotonic() > deadline: + raise + time.sleep(2) + + +def table_count(target): + return int(psql(target, "SELECT count(*) FROM pg_tables WHERE schemaname = 'public' AND tablename <> 'spatial_ref_sys'")) + + +def holds_user_data(target): + """ + A WebUI database that has been started but never used holds only the + schema and boot rows (theme, settings, default node, guardian's anonymous + user). Anything the user created counts as data worth keeping. + """ + try: + rows = psql(target, "SELECT (SELECT count(*) FROM app_project) + (SELECT count(*) FROM app_task) + " + "(SELECT count(*) FROM auth_user WHERE username <> '{}')".format(ANONYMOUS_USER)) + except CommandError: + return True + return int(rows) > 0 + + +def recreate_database(target): + name = target['dbname'] + psql(target, 'DROP DATABASE "{}"'.format(name), dbname='postgres') + psql(target, 'CREATE DATABASE "{}" TEMPLATE template1'.format(name), dbname='postgres') + + +def import_database(source, target, stdout=None): + """ + Copy a WebODM database into a WebUI database that holds no user data. + + source and target are dicts with host, port, user, dbname and an optional + password. Both servers may still be starting up. + """ + wait_for_database(source) + wait_for_database(target) + if table_count(target) > 0: + if holds_user_data(target): + raise CommandError("The target database {} already contains projects, tasks or users. Import only into an unused installation.".format(target['dbname'])) + recreate_database(target) + + dump = subprocess.Popen(['pg_dump'] + pg_args(source) + ['--no-owner', '--no-acl', '--exclude-schema=topology', source['dbname']], + env=pg_env(source), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + restore = subprocess.Popen(['psql'] + pg_args(target) + ['-d', target['dbname'], '-q', '-o', os.devnull, + '-v', 'ON_ERROR_STOP=1', '--single-transaction'], + env=pg_env(target), stdin=dump.stdout, stderr=subprocess.PIPE) + dump.stdout.close() + _, restore_err = restore.communicate() + _, dump_err = dump.communicate() + if restore.returncode != 0: + raise CommandError("restore failed: {}".format(restore_err.decode(errors='replace').strip())) + if dump.returncode != 0: + raise CommandError("pg_dump failed: {}".format(dump_err.decode(errors='replace').strip())) + + for column in ('hostname', 'label'): + psql(target, "UPDATE nodeodm_processingnode SET {0} = '{1}' WHERE {0} = '{2}'".format(column, DEFAULT_NODE_NAME, LEGACY_NODE_NAME)) + + if stdout is not None: + for table, label in (('auth_user', 'users'), ('app_project', 'projects'), ('app_task', 'tasks'), ('nodeodm_processingnode', 'processing nodes')): + stdout.write("{}: {}".format(label, psql(target, "SELECT count(*) FROM {}".format(table)))) + + +class Command(BaseCommand): + help = "Copy the database of a WebODM installation into this unused database. The source password is read from PGPASSWORD or ~/.pgpass." + requires_system_checks = [] + + def add_arguments(self, parser): + parser.add_argument("--host", type=str, required=True, help="Host of the WebODM postgres server") + parser.add_argument("--port", type=int, default=5432, help="Port of the WebODM postgres server") + parser.add_argument("--user", type=str, default="postgres", help="User of the WebODM postgres server") + parser.add_argument("--dbname", type=str, default="webodm_dev", help="Name of the WebODM database") + super().add_arguments(parser) + + def handle(self, **options): + source = { + 'host': options['host'], + 'port': options['port'], + 'user': options['user'], + 'dbname': options['dbname'], + } + db = connection.settings_dict + target = { + 'host': db['HOST'], + 'port': db['PORT'], + 'user': db['USER'], + 'password': db['PASSWORD'], + 'dbname': db['NAME'], + } + import_database(source, target, self.stdout) diff --git a/app/tests/test_importwebodm.py b/app/tests/test_importwebodm.py new file mode 100644 index 00000000..ff56252b --- /dev/null +++ b/app/tests/test_importwebodm.py @@ -0,0 +1,64 @@ +from django.contrib.auth.models import User +from django.core.management.base import CommandError +from django.db import connection +from django.test import TransactionTestCase + +from app.management.commands.importwebodm import import_database, psql +from app.models import Project +from nodeodm.models import ProcessingNode + + +class TestImportWebODM(TransactionTestCase): + """ + Uses the test database as the WebODM source and a scratch database, + created from template1 like the real one, as the WebUI target. + """ + target_name = 'test_importwebodm_target' + + def pg_params(self, dbname): + db = connection.settings_dict + return { + 'host': db['HOST'], + 'port': db['PORT'], + 'user': db['USER'], + 'password': db['PASSWORD'], + 'dbname': dbname, + } + + def setUp(self): + with connection.cursor() as c: + c.execute('DROP DATABASE IF EXISTS {}'.format(self.target_name)) + c.execute('CREATE DATABASE {} TEMPLATE template1'.format(self.target_name)) + self.source = self.pg_params(connection.settings_dict['NAME']) + self.target = self.pg_params(self.target_name) + + user = User.objects.create_user(username='webodm', email='w@o.dm', password='x') + Project.objects.create(owner=user, name='Imported project') + ProcessingNode.objects.create(hostname='node-odx-1', port=3000, label='node-odx-1') + + def tearDown(self): + with connection.cursor() as c: + c.execute('DROP DATABASE IF EXISTS {}'.format(self.target_name)) + + def assertImported(self): + self.assertEqual(psql(self.target, "SELECT name FROM app_project"), 'Imported project') + self.assertEqual(psql(self.target, "SELECT hostname || ' ' || label FROM nodeodm_processingnode"), 'node-odm-1 node-odm-1') + self.assertEqual(psql(self.target, "SELECT count(*) FROM auth_user WHERE username = 'webodm'"), '1') + + def test_import_into_empty_database(self): + import_database(self.source, self.target) + self.assertImported() + + def test_import_replaces_unused_database(self): + # A started but unused WebUI database: tables, boot rows, no user data + psql(self.target, "CREATE TABLE app_project (id int); CREATE TABLE app_task (id int); " + "CREATE TABLE auth_user (username text); INSERT INTO auth_user VALUES ('AnonymousUser'); " + "CREATE TABLE app_theme (id int); INSERT INTO app_theme VALUES (1)") + import_database(self.source, self.target) + self.assertImported() + + def test_import_refuses_database_with_user_data(self): + import_database(self.source, self.target) + with self.assertRaises(CommandError): + import_database(self.source, self.target) + self.assertImported() diff --git a/docker-compose.yml b/docker-compose.yml index 13b301d4..7435b1ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,8 @@ version: '2.1' volumes: dbdata: appmedia: + name: ${WO_MEDIA_VOLUME:-${COMPOSE_PROJECT_NAME}_appmedia} + external: ${WO_MEDIA_EXTERNAL:-false} services: db: image: postgis/postgis:18-3.6-alpine diff --git a/import-webodm.sh b/import-webodm.sh new file mode 100644 index 00000000..3b80968c --- /dev/null +++ b/import-webodm.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Sourced by webui.sh. Imports the database of a WebODM installation into this one. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + echo "Run ./webui.sh import-webodm" + exit 1 +fi + +import_webodm_db_image(){ + $docker_compose -f docker-compose.yml config --images db 2> /dev/null +} + +# WebODM stores a Postgres 9.5 cluster at the root of its database directory or +# volume. Postgres 18 keeps its cluster in 18/docker, so a PG_VERSION file at the +# root of WO_DB_DIR can only be WebODM's. The directory belongs to the postgres +# user with mode 0700, so the check runs in a container. +import_webodm_layout(){ + case "$WO_DB_DIR" in + /*) + if [ -d "$WO_DB_DIR" ] && docker run --rm -v "$WO_DB_DIR:/data:ro" --entrypoint test "$(import_webodm_db_image)" -f /data/PG_VERSION > /dev/null 2>&1; then + echo path + fi + ;; + *) docker volume inspect webodm_dbdata > /dev/null 2>&1 && echo volume ;; + esac + return 0 +} + +import_webodm_project(){ + echo "${COMPOSE_PROJECT_NAME:-$(basename "$__dirname" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_-]//g; s/^[-_]*//')}" +} + +import_webodm_where(){ + if [ "$1" = "path" ]; then + echo "in $WO_DB_DIR" + else + echo "(docker volume webodm_dbdata)" + fi +} + +import_webodm_hint(){ + echo -n "To keep your WebODM projects, tasks and users, run: $0 import-webodm" + [[ $WO_DB_DIR = /* ]] && echo -n " --db-dir $WO_DB_DIR" + [[ $WO_MEDIA_DIR = /* ]] && echo -n " --media-dir $WO_MEDIA_DIR" + echo + if [ "$1" = "path" ]; then + echo "To start with an empty installation instead, pass --db-dir pointing at an empty directory." + else + echo "To start with an empty installation instead, run: $0 start --skip-import" + fi +} + +import_webodm_refuse_dir(){ + echo -e "\033[91mPostgres 18 cannot start in $WO_DB_DIR while it holds a WebODM database.\033[39m" + import_webodm_hint path + exit 1 +} + +import_webodm_check_webodm_stopped(){ + # WebODM and OpenDroneMap WebUI both name their database container "db" + if docker ps --filter name='^db$' --format '{{.Image}}' | grep -q 'webodm_db'; then + echo -e "\033[91mWebODM is running. Stop it first with webodm.sh stop\033[39m" + exit 1 + fi +} + +import_webodm_container="webui-import-webodm-db" +import_webodm_image="webodm/webodm_db" +import_webodm_tmp="" + +# Runs inside a container with WO_DB_DIR mounted at /data after a successful +# import: the 9.5 cluster moves into 9.5/ and the new cluster into 18/. WebODM +# left the directory itself with mode 0700, which the Postgres 18 user cannot +# traverse. +import_webodm_swap=' +cd /data || exit 1 +rmdir 18/docker 18 2> /dev/null +mkdir 9.5 && chmod 755 . || exit 1 +for f in *; do + case "$f" in + 9.5|webui-import) ;; + *) mv "$f" 9.5/ || exit 1 ;; + esac +done +mv webui-import/18 18 && rmdir webui-import +' + +# Run a command showing its output only if it fails +import_webodm_quiet(){ + local output + if ! output=$("$@" 2>&1); then + echo "$output" + import_webodm_cleanup + exit 1 + fi +} + +import_webodm_cleanup(){ + docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true + $docker_compose -f docker-compose.yml rm -sf db > /dev/null 2>&1 + if [ -n "$import_webodm_tmp" ]; then + docker run --rm -v "$WO_DB_DIR:/data" --entrypoint rm "$(import_webodm_db_image)" -rf /data/webui-import > /dev/null 2>&1 || true + fi +} + +# Called by start: offer to import when a WebODM database exists where this installation has never run +import_webodm_offer(){ + local layout answer + layout=$(import_webodm_layout) + if [ -z "$layout" ]; then + return + fi + if [ "$layout" = "volume" ] && docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then + return + fi + import_webodm_check_webodm_stopped + if [[ $skip_import = true ]]; then + [ "$layout" = "volume" ] && return + import_webodm_refuse_dir + fi + if [[ $detached = true ]] || [ ! -t 0 ]; then + echo -e "\033[91mFound a WebODM database $(import_webodm_where "$layout").\033[39m" + import_webodm_hint "$layout" + exit 1 + fi + read -r -p "Found a WebODM database $(import_webodm_where "$layout"). Import it? [Y/n] " answer + if [[ $answer =~ ^[Nn] ]]; then + if [ "$layout" = "volume" ]; then + echo "Starting with an empty installation." + else + import_webodm_refuse_dir + fi + else + import_webodm "$layout" + fi +} + +import_webodm(){ + local layout="${1:-$(import_webodm_layout)}" + local src_db target_db_dir media_volume network output imported + + if [ -z "$layout" ]; then + echo -e "\033[91mNo WebODM database found (docker volume webodm_dbdata, or Postgres data in $WO_DB_DIR).\033[39m" + exit 1 + fi + import_webodm_check_webodm_stopped + + if [ "$layout" = "volume" ]; then + src_db="webodm_dbdata" + target_db_dir="$WO_DB_DIR" + else + src_db="$WO_DB_DIR" + target_db_dir="$WO_DB_DIR/webui-import" + import_webodm_tmp="$target_db_dir" + fi + if [[ $WO_MEDIA_DIR != /* ]] && docker volume inspect webodm_appmedia > /dev/null 2>&1; then + media_volume="webodm_appmedia" + export WO_MEDIA_VOLUME="$media_volume" + export WO_MEDIA_EXTERNAL=true + fi + + echo "Importing from WebODM" + echo "================================" + echo "WebODM database: $src_db" + if [ -n "$media_volume" ]; then + echo "WebODM media: docker volume $media_volume (used in place)" + fi + echo "================================" + + # The WebODM database starts first: its entrypoint chowns everything under + # its data directory, which in the path layout includes the new cluster. + docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true + import_webodm_quiet env WO_DB_DIR="$target_db_dir" $docker_compose -f docker-compose.yml up --no-start db + network=$(docker inspect db --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}') + + echo "Starting the WebODM database..." + import_webodm_quiet docker run -d --name "$import_webodm_container" --network "$network" -v "$src_db:/var/lib/postgresql/data" "$import_webodm_image" + + echo "Starting the OpenDroneMap WebUI database..." + import_webodm_quiet env WO_DB_DIR="$target_db_dir" $docker_compose -f docker-compose.yml up -d db + + echo "Copying the database..." + if output=$($docker_compose -f docker-compose.yml run --rm --no-deps --entrypoint python webapp manage.py importwebodm --host "$import_webodm_container" 2>&1); then + imported=true + else + imported=false + fi + echo "$output" | grep -vE '^\s*$|pkg_resources|Generated secret|Container .* (Creating|Created)|mount of type' || true + if [[ $imported = true ]]; then + import_webodm_tmp="" + fi + import_webodm_cleanup + if [[ $imported != true ]]; then + echo -e "\033[91mImport failed. Your WebODM data is unchanged.\033[39m" + exit 1 + fi + + if [ "$layout" = "path" ]; then + echo "Moving the WebODM database to $WO_DB_DIR/9.5..." + import_webodm_quiet docker run --rm -v "$WO_DB_DIR:/data" --entrypoint sh "$(import_webodm_db_image)" -c "$import_webodm_swap" + fi + if [ -n "$media_volume" ] && ! grep -q '^WO_MEDIA_VOLUME=' "${__dirname}/.env"; then + printf 'WO_MEDIA_VOLUME=%s\nWO_MEDIA_EXTERNAL=true\n' "$media_volume" >> "${__dirname}/.env" + fi + + echo -e "\033[1mDone!\033[0m WebODM data imported." +} diff --git a/webui.sh b/webui.sh index ea01700a..12984285 100755 --- a/webui.sh +++ b/webui.sh @@ -30,6 +30,7 @@ fi # Load default values source "${__dirname}/.env" +source "${__dirname}/import-webodm.sh" DEFAULT_PORT="$WO_PORT" DEFAULT_HOST="$WO_HOST" DEFAULT_MEDIA_DIR="$WO_MEDIA_DIR" @@ -161,6 +162,10 @@ case $key in export WO_IPV6=YES shift # past argument ;; + --skip-import) + skip_import=true + shift # past argument + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -189,6 +194,7 @@ usage(){ echo " rebuild Rebuild all docker containers and perform cleanups" echo " checkenv Do an environment check and install missing components" echo " test [frontend|backend] [args] Run tests (all tests, or just frontend/backend with optional arguments)" + echo " import-webodm [options] Import the database of a stopped WebODM installation into this unused OpenDroneMap WebUI installation and use its media in place" echo " resetadminpassword \"\" Reset the administrator's password to a new one. OpenDroneMap WebUI must be running when executing this command and the password must be enclosed in double quotes." echo "" echo "Options:" @@ -213,6 +219,7 @@ usage(){ echo " --worker-memory Maximum amount of memory allocated for the worker process (default: unlimited)" echo " --worker-cpus Maximum number of CPUs allocated for the worker process (default: all)" echo " --ipv6 Enable IPV6" + echo " --skip-import Start with an empty installation even if a WebODM database is found (default: ask)" exit } @@ -436,6 +443,8 @@ start(){ # mkdir -p $WO_MEDIA_DIR # mkdir -p webui/app/media/tmp + import_webodm_offer + command="$docker_compose -f docker-compose.yml" if [[ $WO_DEFAULT_NODES -gt 0 ]]; then @@ -676,6 +685,9 @@ elif [[ $1 = "checkenv" ]]; then elif [[ $1 = "test" ]]; then shift || true run_tests "$@" +elif [[ $1 = "import-webodm" ]]; then + environment_check + import_webodm elif [[ $1 = "resetadminpassword" ]]; then resetpassword "$2" else From d726c34f314f62005fc7f3c7b3b2edc165216a17 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 28 Oct 2026 10:19:13 +0000 Subject: [PATCH 4/7] Read the stderr of pg_dump and psql directly instead of communicate() On Windows, communicate() starts a reader thread for pg_dump's stdout even though it was closed after being handed to psql, and that thread raises "read of closed file". --- app/management/commands/importwebodm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/management/commands/importwebodm.py b/app/management/commands/importwebodm.py index ad83de94..8c2eebdb 100644 --- a/app/management/commands/importwebodm.py +++ b/app/management/commands/importwebodm.py @@ -92,8 +92,10 @@ def import_database(source, target, stdout=None): '-v', 'ON_ERROR_STOP=1', '--single-transaction'], env=pg_env(target), stdin=dump.stdout, stderr=subprocess.PIPE) dump.stdout.close() - _, restore_err = restore.communicate() - _, dump_err = dump.communicate() + restore_err = restore.stderr.read() + restore.wait() + dump_err = dump.stderr.read() + dump.wait() if restore.returncode != 0: raise CommandError("restore failed: {}".format(restore_err.decode(errors='replace').strip())) if dump.returncode != 0: From 2b9a0e2f215478797818ca7fe49ce9ae784627d2 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 28 Oct 2026 10:57:52 +0000 Subject: [PATCH 5/7] Update README --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d0cebddd..5140acf7 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,8 @@ WebUI is a fully up-to-date stack for generating georeferenced maps, point clouds, elevation models and textured 3D models from images or better yet images + GPS. **📢 OpenDroneMap has officially decoupled from WebODM!** - -WebUI is a distinct product from WebODM, forked from WebODM in order to do the hard work of bringing a web app to users in need of a modern and secure web stack. -**DO NOT install over an existing WebODM installation. Migration scripts forthcoming to do so. Please reach out on https://community.opendronemap.org or info@opendronemap.org if you would like early access to a beta-migration process to upgrade from WebODM** +WebUI is a distinct product from WebODM, forked from WebODM in order to do the hard work of bringing a web app to users in need of a modern and secure web stack. When you run WebUI for the first time, you will have the option to migrate your data across. **See [Up-to-date stack components](#up-to-date-stack-components)** @@ -41,7 +39,7 @@ WebUI is a distinct product from WebODM, forked from WebODM in order to do the h - [Architecture Overview](#architecture-overview) - [License](#license) - [Trademark](#trademark) - + * [Download](https://opendronemap.org/download) * [Documentation](https://docs.opendronemap.org/) * [Hardware Requirements](https://docs.opendronemap.org/installation/#hardware-requirements) @@ -75,7 +73,7 @@ To install WebUI manually on your machine with docker: 2. give Docker enough CPUs (default 2) and RAM (>4Gb, 16Gb better but leave some for Windows) by going to Settings -- Advanced 3. select where on your hard drive you want virtual hard drives to reside (Settings -- Advanced -- Images & Volumes). - + 4. If you want to run the processing component (NodeODM) with GPU acceleration, install [WSL](https://learn.microsoft.com/windows/wsl/) and [set up GPU acceleration](https://learn.microsoft.com/windows/wsl/tutorials/gpu-compute). It's supported on Windows 11 or Windows 10, version 21H2 or higher. ### Installation with Docker @@ -303,7 +301,7 @@ A few things to note: * We use Celery workers to do background tasks such as resizing images and processing task results, but we use an ad-hoc scheduling mechanism to communicate with NodeODM (which processes the orthophotos, 3D models, etc.). The choice to use two separate systems for task scheduling is due to the flexibility that an ad-hoc mechanism gives us for certain operations (capture task output, persistent data and ability to restart tasks mid-way, communication via REST calls, etc.). * If loaded on multiple machines, Celery workers should all share their `app/media` directory with the Django application (via network shares). You can manage workers via `./worker.sh` -# Up-to-date stack components +# Up-to-date stack components In order to support the latest and greatest features, WebUI brings up-to-date many pieces: * Base images @@ -318,7 +316,7 @@ In order to support the latest and greatest features, WebUI brings up-to-date ma * Django 6.0.7 ![Comparison of component versions between WebUI and WebODM](https://github.com/user-attachments/assets/6fa40974-68da-426b-8833-e58877b4c0d6) - + # License OpenDroneMap WebUI is licensed under the terms of the [GNU Affero General Public License v3.0](https://github.com/OpenDroneMap/WebUI/blob/master/LICENSE.md). From 8889d20147136062578fb2261158c31d0dc028bc Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 28 Oct 2026 15:37:38 +0000 Subject: [PATCH 6/7] Prefix container names with webui- WebODM and WebUI used the same container names (db, webapp, worker, broker), so a WebODM installation stopped with webodm.sh stop blocked WebUI from creating its containers. Service names are unchanged, so the hostnames stored in the database (db, broker, node-odm-1) still resolve. The import checks for a running WebODM database by image, not by name. --- docker-compose.nodemicmac.yml | 2 +- docker-compose.nodeodm.gpu.nvidia.yml | 2 +- docker-compose.nodeodm.yml | 2 +- docker-compose.yml | 8 ++++---- import-webodm.sh | 5 ++--- webui.sh | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docker-compose.nodemicmac.yml b/docker-compose.nodemicmac.yml index bc978a95..0d3e10c3 100644 --- a/docker-compose.nodemicmac.yml +++ b/docker-compose.nodemicmac.yml @@ -11,7 +11,7 @@ services: - WO_CREATE_MICMAC_PNODE=YES node-micmac-1: image: opendronemap/nodemicmac - container_name: node-micmac-1 + container_name: webui-node-micmac-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.nodeodm.gpu.nvidia.yml b/docker-compose.nodeodm.gpu.nvidia.yml index c35ac7be..03dd2e0d 100644 --- a/docker-compose.nodeodm.gpu.nvidia.yml +++ b/docker-compose.nodeodm.gpu.nvidia.yml @@ -11,7 +11,7 @@ services: - WO_DEFAULT_NODES node-odm-1: image: opendronemap/nodeodm:3.6.2-gpu - container_name: node-odm-1 + container_name: webui-node-odm-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 79224466..a5a8a81c 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -11,7 +11,7 @@ services: - WO_DEFAULT_NODES node-odm-1: image: opendronemap/nodeodm:3.6.2 - container_name: node-odm-1 + container_name: webui-node-odm-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 7435b1ae..8e521e6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ volumes: services: db: image: postgis/postgis:18-3.6-alpine - container_name: db + container_name: webui-db expose: - "5432" volumes: @@ -32,7 +32,7 @@ services: oom_score_adj: -100 webapp: image: opendronemap/webui - container_name: webapp + container_name: webui-webapp entrypoint: /bin/bash -c "crond && chmod +x /webui/*.sh && /bin/bash -c \"/webui/wait-for-postgres.sh db /webui/wait-for-it.sh -t 0 broker:6379 -- /webui/start.sh\"" volumes: - ${WO_MEDIA_DIR}:/webui/app/media:z @@ -56,12 +56,12 @@ services: oom_score_adj: 0 broker: image: valkey/valkey:9.1.0-alpine3.23 - container_name: broker + container_name: webui-broker restart: unless-stopped oom_score_adj: -500 worker: image: opendronemap/webui - container_name: worker + container_name: webui-worker entrypoint: /bin/bash -c "/webui/wait-for-postgres.sh db /webui/wait-for-it.sh -t 0 broker:6379 -- /webui/wait-for-it.sh -t 0 webapp:8000 -- /webui/worker.sh start" volumes: - ${WO_MEDIA_DIR}:/webui/app/media:z diff --git a/import-webodm.sh b/import-webodm.sh index 3b80968c..f34db0d0 100644 --- a/import-webodm.sh +++ b/import-webodm.sh @@ -56,8 +56,7 @@ import_webodm_refuse_dir(){ } import_webodm_check_webodm_stopped(){ - # WebODM and OpenDroneMap WebUI both name their database container "db" - if docker ps --filter name='^db$' --format '{{.Image}}' | grep -q 'webodm_db'; then + if docker ps --format '{{.Image}}' | grep -q '^webodm/webodm_db'; then echo -e "\033[91mWebODM is running. Stop it first with webodm.sh stop\033[39m" exit 1 fi @@ -170,7 +169,7 @@ import_webodm(){ # its data directory, which in the path layout includes the new cluster. docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true import_webodm_quiet env WO_DB_DIR="$target_db_dir" $docker_compose -f docker-compose.yml up --no-start db - network=$(docker inspect db --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}') + network=$(docker inspect webui-db --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}') echo "Starting the WebODM database..." import_webodm_quiet docker run -d --name "$import_webodm_container" --network "$network" -v "$src_db:/var/lib/postgresql/data" "$import_webodm_image" diff --git a/webui.sh b/webui.sh index 12984285..12770663 100755 --- a/webui.sh +++ b/webui.sh @@ -590,7 +590,7 @@ resetpassword(){ newpass=$1 if [[ -n "$newpass" ]]; then - container_hash=$(docker ps -q --filter "name=webapp") + container_hash=$(docker ps -q --filter "name=^webui-webapp$") if [[ -z "$container_hash" ]]; then echo -e "\033[91mCannot find webapp docker container. Is OpenDroneMap WebUI running?\033[39m" exit 1 From aa49fb2eb79a5f97ef48b3ffba33fb5d5b916ab5 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 28 Oct 2026 15:37:46 +0000 Subject: [PATCH 7/7] Import a WebODM database into a separate directory or volume Both layouts now copy from a source into a target that this run owns: the docker volume webodm_dbdata into this installation's volume, or the directory given with --webodm-db-dir into the one given with --db-dir (or the volume). The WebODM database is never written to. A start on a WebODM database directory is refused with the import command. Previously the path layout started Postgres 18 inside the WebODM directory and swapped the two clusters afterwards, which needed a temporary directory, its removal on failure, a leftover check and a wait for the WebODM entrypoint's chown to finish. A failed import removes the database volume it created, so the next start offers the import again instead of starting empty. docker run is called with MSYS_NO_PATHCONV=1 where container paths are passed, so Git Bash does not rewrite them into Windows paths. --- README.md | 13 +++ import-webodm.sh | 225 ++++++++++++++++++++++------------------------- webui.sh | 10 ++- 3 files changed, 126 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 5140acf7..b36d07bd 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,19 @@ For various reasons such as ease of backup/restore, if you want to store your fi Note that existing task results will not be available after the change. Refer to the [Migrate Data Volumes](https://docs.docker.com/engine/tutorials/dockervolumes/#backup-restore-or-migrate-data-volumes) section of the Docker documentation for information on migrating existing task results. +### Migrating from WebODM + +Stop WebODM with `./webodm.sh stop`. If WebODM kept its database in a docker volume (the default), the first `./webui.sh start` finds it and offers to import it. The projects, tasks and users are copied into WebUI's own database and WebODM's media volume is used in place, so no task results are copied. WebODM's database is not modified. + +If WebODM was started with `--db-dir`, Postgres 18 cannot use that directory. Import it into a new one: + +```bash +./webui.sh import-webodm --webodm-db-dir /home/user/webodm_db --db-dir /home/user/webui_db +./webui.sh start --db-dir /home/user/webui_db +``` + +Leave out `--db-dir` to import into a docker volume instead. + ### Common Troubleshooting Symptoms | Possible Solutions diff --git a/import-webodm.sh b/import-webodm.sh index f34db0d0..703f4af5 100644 --- a/import-webodm.sh +++ b/import-webodm.sh @@ -5,56 +5,44 @@ if [[ ${BASH_SOURCE[0]} == "$0" ]]; then exit 1 fi +import_webodm_container="webui-import-webodm-db" +import_webodm_image="webodm/webodm_db" +import_webodm_new_volume="" + +# Git Bash rewrites arguments that look like POSIX paths (/data/PG_VERSION, +# /c/dir:/data) into Windows paths before docker sees them. MSYS_NO_PATHCONV +# turns that off for one command and is ignored everywhere else. +import_webodm_docker_run(){ + MSYS_NO_PATHCONV=1 docker run "$@" +} + import_webodm_db_image(){ $docker_compose -f docker-compose.yml config --images db 2> /dev/null } -# WebODM stores a Postgres 9.5 cluster at the root of its database directory or -# volume. Postgres 18 keeps its cluster in 18/docker, so a PG_VERSION file at the -# root of WO_DB_DIR can only be WebODM's. The directory belongs to the postgres -# user with mode 0700, so the check runs in a container. -import_webodm_layout(){ - case "$WO_DB_DIR" in - /*) - if [ -d "$WO_DB_DIR" ] && docker run --rm -v "$WO_DB_DIR:/data:ro" --entrypoint test "$(import_webodm_db_image)" -f /data/PG_VERSION > /dev/null 2>&1; then - echo path - fi - ;; - *) docker volume inspect webodm_dbdata > /dev/null 2>&1 && echo volume ;; - esac - return 0 +# WebODM keeps a Postgres 9.5 cluster at the root of its database directory. +# Postgres 18 keeps its cluster in 18/docker, so a PG_VERSION file at the root +# can only be WebODM's. The directory belongs to the postgres user with mode +# 0700, so the check runs in a container. +import_webodm_is_webodm_dir(){ + [ -d "$1" ] && import_webodm_docker_run --rm -v "$1:/data:ro" --entrypoint test "$(import_webodm_db_image)" -f /data/PG_VERSION > /dev/null 2>&1 } import_webodm_project(){ echo "${COMPOSE_PROJECT_NAME:-$(basename "$__dirname" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_-]//g; s/^[-_]*//')}" } -import_webodm_where(){ - if [ "$1" = "path" ]; then - echo "in $WO_DB_DIR" - else - echo "(docker volume webodm_dbdata)" - fi -} - -import_webodm_hint(){ - echo -n "To keep your WebODM projects, tasks and users, run: $0 import-webodm" +import_webodm_dir_flags(){ [[ $WO_DB_DIR = /* ]] && echo -n " --db-dir $WO_DB_DIR" [[ $WO_MEDIA_DIR = /* ]] && echo -n " --media-dir $WO_MEDIA_DIR" - echo - if [ "$1" = "path" ]; then - echo "To start with an empty installation instead, pass --db-dir pointing at an empty directory." - else - echo "To start with an empty installation instead, run: $0 start --skip-import" - fi + return 0 } -import_webodm_refuse_dir(){ - echo -e "\033[91mPostgres 18 cannot start in $WO_DB_DIR while it holds a WebODM database.\033[39m" - import_webodm_hint path - exit 1 +import_webodm_start_hint(){ + echo "Start OpenDroneMap WebUI with: $0 start$(import_webodm_dir_flags)" } +# Two Postgres servers on one data directory corrupt it import_webodm_check_webodm_stopped(){ if docker ps --format '{{.Image}}' | grep -q '^webodm/webodm_db'; then echo -e "\033[91mWebODM is running. Stop it first with webodm.sh stop\033[39m" @@ -62,95 +50,102 @@ import_webodm_check_webodm_stopped(){ fi } -import_webodm_container="webui-import-webodm-db" -import_webodm_image="webodm/webodm_db" -import_webodm_tmp="" - -# Runs inside a container with WO_DB_DIR mounted at /data after a successful -# import: the 9.5 cluster moves into 9.5/ and the new cluster into 18/. WebODM -# left the directory itself with mode 0700, which the Postgres 18 user cannot -# traverse. -import_webodm_swap=' -cd /data || exit 1 -rmdir 18/docker 18 2> /dev/null -mkdir 9.5 && chmod 755 . || exit 1 -for f in *; do - case "$f" in - 9.5|webui-import) ;; - *) mv "$f" 9.5/ || exit 1 ;; - esac -done -mv webui-import/18 18 && rmdir webui-import -' - -# Run a command showing its output only if it fails -import_webodm_quiet(){ - local output - if ! output=$("$@" 2>&1); then - echo "$output" - import_webodm_cleanup - exit 1 - fi -} - -import_webodm_cleanup(){ - docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true - $docker_compose -f docker-compose.yml rm -sf db > /dev/null 2>&1 - if [ -n "$import_webodm_tmp" ]; then - docker run --rm -v "$WO_DB_DIR:/data" --entrypoint rm "$(import_webodm_db_image)" -rf /data/webui-import > /dev/null 2>&1 || true - fi -} - -# Called by start: offer to import when a WebODM database exists where this installation has never run +# Called by start. A WebODM database in a docker volume is offered for import +# until this installation's own volume exists. Postgres 18 cannot use a WebODM +# database directory, so a start on one is refused with the import command. import_webodm_offer(){ - local layout answer - layout=$(import_webodm_layout) - if [ -z "$layout" ]; then + local answer + if [[ $WO_DB_DIR = /* ]]; then + if import_webodm_is_webodm_dir "$WO_DB_DIR"; then + echo -e "\033[91mPostgres 18 cannot start in $WO_DB_DIR: it holds a WebODM database.\033[39m" + echo "To keep your WebODM projects, tasks and users, run: $0 import-webodm --webodm-db-dir $WO_DB_DIR --db-dir " + echo "Leave out --db-dir to import into a docker volume instead." + echo "To start with an empty installation, pass --db-dir pointing at an empty directory." + exit 1 + fi return fi - if [ "$layout" = "volume" ] && docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then + if ! docker volume inspect webodm_dbdata > /dev/null 2>&1 || docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then return fi import_webodm_check_webodm_stopped if [[ $skip_import = true ]]; then - [ "$layout" = "volume" ] && return - import_webodm_refuse_dir + return fi if [[ $detached = true ]] || [ ! -t 0 ]; then - echo -e "\033[91mFound a WebODM database $(import_webodm_where "$layout").\033[39m" - import_webodm_hint "$layout" + echo -e "\033[91mFound a WebODM database (docker volume webodm_dbdata).\033[39m" + echo "To keep your WebODM projects, tasks and users, run: $0 import-webodm$(import_webodm_dir_flags)" + echo "To start with an empty installation instead, run: $0 start --skip-import" exit 1 fi - read -r -p "Found a WebODM database $(import_webodm_where "$layout"). Import it? [Y/n] " answer + read -r -p "Found a WebODM database (docker volume webodm_dbdata). Import it? [Y/n] " answer if [[ $answer =~ ^[Nn] ]]; then - if [ "$layout" = "volume" ]; then - echo "Starting with an empty installation." - else - import_webodm_refuse_dir - fi + echo "Starting with an empty installation." else - import_webodm "$layout" + import_webodm + fi +} + +# Run a command showing its output only if it fails +import_webodm_quiet(){ + local output + if ! output=$("$@" 2>&1); then + echo "$output" + return 1 fi } +import_webodm_stop_servers(){ + docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true + $docker_compose -f docker-compose.yml rm -sf db > /dev/null 2>&1 || true +} + +# Exit trap while the import runs. The database volume this run created goes +# too, so the next start offers the import again. +import_webodm_failed(){ + trap - EXIT + import_webodm_stop_servers + if [ -n "$import_webodm_new_volume" ]; then + docker volume rm "$import_webodm_new_volume" > /dev/null 2>&1 || true + fi + echo -e "\033[91mImport failed. Your WebODM data is unchanged.\033[39m" + exit 1 +} + +# Copies the WebODM database (docker volume webodm_dbdata, or the directory +# given with --webodm-db-dir) into this installation's database (its docker +# volume, or the directory given with --db-dir). WebODM's media volume is +# reused in place. Nothing of WebODM's is modified. import_webodm(){ - local layout="${1:-$(import_webodm_layout)}" - local src_db target_db_dir media_volume network output imported + local source target network output imported media_volume - if [ -z "$layout" ]; then - echo -e "\033[91mNo WebODM database found (docker volume webodm_dbdata, or Postgres data in $WO_DB_DIR).\033[39m" + if [ -n "$WO_WEBODM_DB_DIR" ]; then + source="$WO_WEBODM_DB_DIR" + if ! import_webodm_is_webodm_dir "$source"; then + echo -e "\033[91mNo WebODM database in $source.\033[39m" + exit 1 + fi + elif docker volume inspect webodm_dbdata > /dev/null 2>&1; then + source="webodm_dbdata" + else + echo -e "\033[91mNo WebODM database found (docker volume webodm_dbdata). For a WebODM started with --db-dir, pass --webodm-db-dir.\033[39m" exit 1 fi - import_webodm_check_webodm_stopped - if [ "$layout" = "volume" ]; then - src_db="webodm_dbdata" - target_db_dir="$WO_DB_DIR" + if [[ $WO_DB_DIR = /* ]]; then + if [ "$WO_DB_DIR" = "$source" ] || import_webodm_is_webodm_dir "$WO_DB_DIR"; then + echo -e "\033[91m$WO_DB_DIR holds a WebODM database. Pass --db-dir pointing at a new directory.\033[39m" + exit 1 + fi + target="$WO_DB_DIR" else - src_db="$WO_DB_DIR" - target_db_dir="$WO_DB_DIR/webui-import" - import_webodm_tmp="$target_db_dir" + target="docker volume $(import_webodm_project)_dbdata" + if ! docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then + import_webodm_new_volume="$(import_webodm_project)_dbdata" + fi fi + import_webodm_check_webodm_stopped + if [[ $WO_MEDIA_DIR != /* ]] && docker volume inspect webodm_appmedia > /dev/null 2>&1; then media_volume="webodm_appmedia" export WO_MEDIA_VOLUME="$media_volume" @@ -159,47 +154,35 @@ import_webodm(){ echo "Importing from WebODM" echo "================================" - echo "WebODM database: $src_db" + echo "WebODM database: $source" + echo "OpenDroneMap WebUI database: $target" if [ -n "$media_volume" ]; then echo "WebODM media: docker volume $media_volume (used in place)" fi echo "================================" - # The WebODM database starts first: its entrypoint chowns everything under - # its data directory, which in the path layout includes the new cluster. + trap import_webodm_failed EXIT docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true - import_webodm_quiet env WO_DB_DIR="$target_db_dir" $docker_compose -f docker-compose.yml up --no-start db + import_webodm_quiet $docker_compose -f docker-compose.yml up --no-start db network=$(docker inspect webui-db --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}') echo "Starting the WebODM database..." - import_webodm_quiet docker run -d --name "$import_webodm_container" --network "$network" -v "$src_db:/var/lib/postgresql/data" "$import_webodm_image" + import_webodm_quiet import_webodm_docker_run -d --name "$import_webodm_container" --network "$network" -v "$source:/var/lib/postgresql/data" "$import_webodm_image" echo "Starting the OpenDroneMap WebUI database..." - import_webodm_quiet env WO_DB_DIR="$target_db_dir" $docker_compose -f docker-compose.yml up -d db + import_webodm_quiet $docker_compose -f docker-compose.yml up -d db echo "Copying the database..." - if output=$($docker_compose -f docker-compose.yml run --rm --no-deps --entrypoint python webapp manage.py importwebodm --host "$import_webodm_container" 2>&1); then - imported=true - else - imported=false - fi + output=$($docker_compose -f docker-compose.yml run --rm --no-deps --entrypoint python webapp manage.py importwebodm --host "$import_webodm_container" 2>&1) && imported=true || imported=false echo "$output" | grep -vE '^\s*$|pkg_resources|Generated secret|Container .* (Creating|Created)|mount of type' || true - if [[ $imported = true ]]; then - import_webodm_tmp="" - fi - import_webodm_cleanup if [[ $imported != true ]]; then - echo -e "\033[91mImport failed. Your WebODM data is unchanged.\033[39m" exit 1 fi - if [ "$layout" = "path" ]; then - echo "Moving the WebODM database to $WO_DB_DIR/9.5..." - import_webodm_quiet docker run --rm -v "$WO_DB_DIR:/data" --entrypoint sh "$(import_webodm_db_image)" -c "$import_webodm_swap" - fi + trap - EXIT + import_webodm_stop_servers if [ -n "$media_volume" ] && ! grep -q '^WO_MEDIA_VOLUME=' "${__dirname}/.env"; then printf 'WO_MEDIA_VOLUME=%s\nWO_MEDIA_EXTERNAL=true\n' "$media_volume" >> "${__dirname}/.env" fi - echo -e "\033[1mDone!\033[0m WebODM data imported." } diff --git a/webui.sh b/webui.sh index 12770663..369b90ad 100755 --- a/webui.sh +++ b/webui.sh @@ -166,6 +166,12 @@ case $key in skip_import=true shift # past argument ;; + --webodm-db-dir) + WO_WEBODM_DB_DIR=$(realpath "$2") + export WO_WEBODM_DB_DIR + shift # past argument + shift # past value + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -194,7 +200,7 @@ usage(){ echo " rebuild Rebuild all docker containers and perform cleanups" echo " checkenv Do an environment check and install missing components" echo " test [frontend|backend] [args] Run tests (all tests, or just frontend/backend with optional arguments)" - echo " import-webodm [options] Import the database of a stopped WebODM installation into this unused OpenDroneMap WebUI installation and use its media in place" + echo " import-webodm [options] Import the database of a stopped WebODM installation into this unused OpenDroneMap WebUI installation and use its media in place. Pass --webodm-db-dir for a WebODM started with --db-dir." echo " resetadminpassword \"\" Reset the administrator's password to a new one. OpenDroneMap WebUI must be running when executing this command and the password must be enclosed in double quotes." echo "" echo "Options:" @@ -220,6 +226,7 @@ usage(){ echo " --worker-cpus Maximum number of CPUs allocated for the worker process (default: all)" echo " --ipv6 Enable IPV6" echo " --skip-import Start with an empty installation even if a WebODM database is found (default: ask)" + echo " --webodm-db-dir Database directory of a WebODM installation started with --db-dir, for import-webodm (default: docker volume webodm_dbdata)" exit } @@ -688,6 +695,7 @@ elif [[ $1 = "test" ]]; then elif [[ $1 = "import-webodm" ]]; then environment_check import_webodm + import_webodm_start_hint elif [[ $1 = "resetadminpassword" ]]; then resetpassword "$2" else