diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 32325bbd..2e51e4f8 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -29,6 +29,7 @@ jobs: docker compose -f docker-compose.yml -f docker-compose.build.yml build --build-arg TEST_BUILD=ON docker compose -f docker-compose.yml -f docker-compose.build.yml up -d sleep 20 + docker compose exec -T webapp python manage.py makemigrations --check --dry-run docker compose exec -T webapp /webui/webui.sh test image: diff --git a/README.md b/README.md index d0cebddd..b36d07bd 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,8 @@ WebUI is a fully up-to-date stack for generating georeferenced maps, point clouds, elevation models and textured 3D models from images or better yet images + GPS. **📢 OpenDroneMap has officially decoupled from WebODM!** - -WebUI is a distinct product from WebODM, forked from WebODM in order to do the hard work of bringing a web app to users in need of a modern and secure web stack. -**DO NOT install over an existing WebODM installation. Migration scripts forthcoming to do so. Please reach out on https://community.opendronemap.org or info@opendronemap.org if you would like early access to a beta-migration process to upgrade from WebODM** +WebUI is a distinct product from WebODM, forked from WebODM in order to do the hard work of bringing a web app to users in need of a modern and secure web stack. When you run WebUI for the first time, you will have the option to migrate your data across. **See [Up-to-date stack components](#up-to-date-stack-components)** @@ -41,7 +39,7 @@ WebUI is a distinct product from WebODM, forked from WebODM in order to do the h - [Architecture Overview](#architecture-overview) - [License](#license) - [Trademark](#trademark) - + * [Download](https://opendronemap.org/download) * [Documentation](https://docs.opendronemap.org/) * [Hardware Requirements](https://docs.opendronemap.org/installation/#hardware-requirements) @@ -75,7 +73,7 @@ To install WebUI manually on your machine with docker: 2. give Docker enough CPUs (default 2) and RAM (>4Gb, 16Gb better but leave some for Windows) by going to Settings -- Advanced 3. select where on your hard drive you want virtual hard drives to reside (Settings -- Advanced -- Images & Volumes). - + 4. If you want to run the processing component (NodeODM) with GPU acceleration, install [WSL](https://learn.microsoft.com/windows/wsl/) and [set up GPU acceleration](https://learn.microsoft.com/windows/wsl/tutorials/gpu-compute). It's supported on Windows 11 or Windows 10, version 21H2 or higher. ### Installation with Docker @@ -191,6 +189,19 @@ For various reasons such as ease of backup/restore, if you want to store your fi Note that existing task results will not be available after the change. Refer to the [Migrate Data Volumes](https://docs.docker.com/engine/tutorials/dockervolumes/#backup-restore-or-migrate-data-volumes) section of the Docker documentation for information on migrating existing task results. +### Migrating from WebODM + +Stop WebODM with `./webodm.sh stop`. If WebODM kept its database in a docker volume (the default), the first `./webui.sh start` finds it and offers to import it. The projects, tasks and users are copied into WebUI's own database and WebODM's media volume is used in place, so no task results are copied. WebODM's database is not modified. + +If WebODM was started with `--db-dir`, Postgres 18 cannot use that directory. Import it into a new one: + +```bash +./webui.sh import-webodm --webodm-db-dir /home/user/webodm_db --db-dir /home/user/webui_db +./webui.sh start --db-dir /home/user/webui_db +``` + +Leave out `--db-dir` to import into a docker volume instead. + ### Common Troubleshooting Symptoms | Possible Solutions @@ -303,7 +314,7 @@ A few things to note: * We use Celery workers to do background tasks such as resizing images and processing task results, but we use an ad-hoc scheduling mechanism to communicate with NodeODM (which processes the orthophotos, 3D models, etc.). The choice to use two separate systems for task scheduling is due to the flexibility that an ad-hoc mechanism gives us for certain operations (capture task output, persistent data and ability to restart tasks mid-way, communication via REST calls, etc.). * If loaded on multiple machines, Celery workers should all share their `app/media` directory with the Django application (via network shares). You can manage workers via `./worker.sh` -# Up-to-date stack components +# Up-to-date stack components In order to support the latest and greatest features, WebUI brings up-to-date many pieces: * Base images @@ -318,7 +329,7 @@ In order to support the latest and greatest features, WebUI brings up-to-date ma * Django 6.0.7 ![Comparison of component versions between WebUI and WebODM](https://github.com/user-attachments/assets/6fa40974-68da-426b-8833-e58877b4c0d6) - + # License OpenDroneMap WebUI is licensed under the terms of the [GNU Affero General Public License v3.0](https://github.com/OpenDroneMap/WebUI/blob/master/LICENSE.md). diff --git a/app/management/commands/importwebodm.py b/app/management/commands/importwebodm.py new file mode 100644 index 00000000..8c2eebdb --- /dev/null +++ b/app/management/commands/importwebodm.py @@ -0,0 +1,138 @@ +import os +import subprocess +import time + +from django.core.management.base import BaseCommand, CommandError +from django.db import connection + +DEFAULT_NODE_NAME = 'node-odm-1' +LEGACY_NODE_NAME = 'node-odx-1' +ANONYMOUS_USER = 'AnonymousUser' + + +def pg_args(params): + args = [] + if params.get('host'): + args += ['-h', params['host']] + if params.get('port'): + args += ['-p', str(params['port'])] + if params.get('user'): + args += ['-U', params['user']] + return args + + +def pg_env(params): + env = os.environ.copy() + if params.get('password'): + env['PGPASSWORD'] = params['password'] + return env + + +def psql(params, sql, dbname=None): + result = subprocess.run(['psql'] + pg_args(params) + ['-d', dbname or params['dbname'], '-tAc', sql], + env=pg_env(params), capture_output=True, text=True) + if result.returncode != 0: + raise CommandError("psql failed: {}".format(result.stderr.strip())) + return result.stdout.strip() + + +def wait_for_database(params, timeout=120): + deadline = time.monotonic() + timeout + while True: + try: + psql(params, 'SELECT 1') + return + except CommandError: + if time.monotonic() > deadline: + raise + time.sleep(2) + + +def table_count(target): + return int(psql(target, "SELECT count(*) FROM pg_tables WHERE schemaname = 'public' AND tablename <> 'spatial_ref_sys'")) + + +def holds_user_data(target): + """ + A WebUI database that has been started but never used holds only the + schema and boot rows (theme, settings, default node, guardian's anonymous + user). Anything the user created counts as data worth keeping. + """ + try: + rows = psql(target, "SELECT (SELECT count(*) FROM app_project) + (SELECT count(*) FROM app_task) + " + "(SELECT count(*) FROM auth_user WHERE username <> '{}')".format(ANONYMOUS_USER)) + except CommandError: + return True + return int(rows) > 0 + + +def recreate_database(target): + name = target['dbname'] + psql(target, 'DROP DATABASE "{}"'.format(name), dbname='postgres') + psql(target, 'CREATE DATABASE "{}" TEMPLATE template1'.format(name), dbname='postgres') + + +def import_database(source, target, stdout=None): + """ + Copy a WebODM database into a WebUI database that holds no user data. + + source and target are dicts with host, port, user, dbname and an optional + password. Both servers may still be starting up. + """ + wait_for_database(source) + wait_for_database(target) + if table_count(target) > 0: + if holds_user_data(target): + raise CommandError("The target database {} already contains projects, tasks or users. Import only into an unused installation.".format(target['dbname'])) + recreate_database(target) + + dump = subprocess.Popen(['pg_dump'] + pg_args(source) + ['--no-owner', '--no-acl', '--exclude-schema=topology', source['dbname']], + env=pg_env(source), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + restore = subprocess.Popen(['psql'] + pg_args(target) + ['-d', target['dbname'], '-q', '-o', os.devnull, + '-v', 'ON_ERROR_STOP=1', '--single-transaction'], + env=pg_env(target), stdin=dump.stdout, stderr=subprocess.PIPE) + dump.stdout.close() + restore_err = restore.stderr.read() + restore.wait() + dump_err = dump.stderr.read() + dump.wait() + if restore.returncode != 0: + raise CommandError("restore failed: {}".format(restore_err.decode(errors='replace').strip())) + if dump.returncode != 0: + raise CommandError("pg_dump failed: {}".format(dump_err.decode(errors='replace').strip())) + + for column in ('hostname', 'label'): + psql(target, "UPDATE nodeodm_processingnode SET {0} = '{1}' WHERE {0} = '{2}'".format(column, DEFAULT_NODE_NAME, LEGACY_NODE_NAME)) + + if stdout is not None: + for table, label in (('auth_user', 'users'), ('app_project', 'projects'), ('app_task', 'tasks'), ('nodeodm_processingnode', 'processing nodes')): + stdout.write("{}: {}".format(label, psql(target, "SELECT count(*) FROM {}".format(table)))) + + +class Command(BaseCommand): + help = "Copy the database of a WebODM installation into this unused database. The source password is read from PGPASSWORD or ~/.pgpass." + requires_system_checks = [] + + def add_arguments(self, parser): + parser.add_argument("--host", type=str, required=True, help="Host of the WebODM postgres server") + parser.add_argument("--port", type=int, default=5432, help="Port of the WebODM postgres server") + parser.add_argument("--user", type=str, default="postgres", help="User of the WebODM postgres server") + parser.add_argument("--dbname", type=str, default="webodm_dev", help="Name of the WebODM database") + super().add_arguments(parser) + + def handle(self, **options): + source = { + 'host': options['host'], + 'port': options['port'], + 'user': options['user'], + 'dbname': options['dbname'], + } + db = connection.settings_dict + target = { + 'host': db['HOST'], + 'port': db['PORT'], + 'user': db['USER'], + 'password': db['PASSWORD'], + 'dbname': db['NAME'], + } + import_database(source, target, self.stdout) diff --git a/app/migrations/0054_alter_setting_organization_name_and_more.py b/app/migrations/0054_alter_setting_organization_name_and_more.py new file mode 100644 index 00000000..6f0e6ae1 --- /dev/null +++ b/app/migrations/0054_alter_setting_organization_name_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 6.0.7 on 2026-10-26 11:56 + +import colorfield.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0053_alter_basemap_id_alter_plugindatum_id_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='setting', + name='organization_name', + field=models.CharField(blank=True, default='OpenDroneMap', help_text='The name of your organization', max_length=255, null=True, verbose_name='Organization name'), + ), + migrations.AlterField( + model_name='setting', + name='organization_website', + field=models.URLField(blank=True, default='https://opendronemap.org', help_text='The website URL of your organization', max_length=255, null=True, verbose_name='Organization website'), + ), + migrations.AlterField( + model_name='theme', + name='button_danger', + field=colorfield.fields.ColorField(default='#cb4b16', help_text='Delete button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Danger'), + ), + migrations.AlterField( + model_name='theme', + name='button_default', + field=colorfield.fields.ColorField(default='#60818e', help_text='Default button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Default'), + ), + migrations.AlterField( + model_name='theme', + name='button_primary', + field=colorfield.fields.ColorField(default='#07485b', help_text='Primary button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Primary'), + ), + migrations.AlterField( + model_name='theme', + name='dialog_warning', + field=colorfield.fields.ColorField(default='#cb4b16', help_text='The border color of warning dialogs.', image_field=None, max_length=25, samples=None, verbose_name='Dialog Warning'), + ), + migrations.AlterField( + model_name='theme', + name='header_background', + field=colorfield.fields.ColorField(default='#60818e', help_text="Background color of the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Background'), + ), + migrations.AlterField( + model_name='theme', + name='header_primary', + field=colorfield.fields.ColorField(default='#ecf9fb', help_text="Text and icons in the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Primary'), + ), + migrations.AlterField( + model_name='theme', + name='highlight', + field=colorfield.fields.ColorField(default='#eee8d5', help_text='The background color of panels and some borders.', image_field=None, max_length=25, samples=None, verbose_name='Highlight'), + ), + migrations.AlterField( + model_name='theme', + name='primary', + field=colorfield.fields.ColorField(default='#07485b', help_text='Most text, icons, and borders.', image_field=None, max_length=25, samples=None, verbose_name='Primary'), + ), + migrations.AlterField( + model_name='theme', + name='secondary', + field=colorfield.fields.ColorField(default='#fef5e6', help_text='The main background color, and text color of some buttons.', image_field=None, max_length=25, samples=None, verbose_name='Secondary'), + ), + migrations.AlterField( + model_name='theme', + name='success', + field=colorfield.fields.ColorField(default='#cbd198', help_text='The background color of success notifications.', image_field=None, max_length=25, samples=None, verbose_name='Success'), + ), + migrations.AlterField( + model_name='theme', + name='tertiary', + field=colorfield.fields.ColorField(default='#0d6986', help_text='Navigation links.', image_field=None, max_length=25, samples=None, verbose_name='Tertiary'), + ), + ] diff --git a/app/models/README.md b/app/models/README.md new file mode 100644 index 00000000..8d31e4a8 --- /dev/null +++ b/app/models/README.md @@ -0,0 +1,19 @@ +# Models and migrations + +The classes in this directory and in `nodeodm/models.py` define the database tables. + +After changing a model generate the matching migration and commit it with the change: + +``` +docker compose exec webapp python manage.py makemigrations +``` + +The command writes a new numbered file under the app's `migrations/` directory. + +Migrations are applied automatically by `start.sh` when the webapp container starts. Nothing needs to run `migrate` by hand. + +Two branches that each add a migration with the same parent produce the error `Conflicting migrations detected; multiple leaf nodes`. Resolve it with: + +``` +docker compose exec webapp python manage.py makemigrations --merge +``` diff --git a/app/tests/test_importwebodm.py b/app/tests/test_importwebodm.py new file mode 100644 index 00000000..ff56252b --- /dev/null +++ b/app/tests/test_importwebodm.py @@ -0,0 +1,64 @@ +from django.contrib.auth.models import User +from django.core.management.base import CommandError +from django.db import connection +from django.test import TransactionTestCase + +from app.management.commands.importwebodm import import_database, psql +from app.models import Project +from nodeodm.models import ProcessingNode + + +class TestImportWebODM(TransactionTestCase): + """ + Uses the test database as the WebODM source and a scratch database, + created from template1 like the real one, as the WebUI target. + """ + target_name = 'test_importwebodm_target' + + def pg_params(self, dbname): + db = connection.settings_dict + return { + 'host': db['HOST'], + 'port': db['PORT'], + 'user': db['USER'], + 'password': db['PASSWORD'], + 'dbname': dbname, + } + + def setUp(self): + with connection.cursor() as c: + c.execute('DROP DATABASE IF EXISTS {}'.format(self.target_name)) + c.execute('CREATE DATABASE {} TEMPLATE template1'.format(self.target_name)) + self.source = self.pg_params(connection.settings_dict['NAME']) + self.target = self.pg_params(self.target_name) + + user = User.objects.create_user(username='webodm', email='w@o.dm', password='x') + Project.objects.create(owner=user, name='Imported project') + ProcessingNode.objects.create(hostname='node-odx-1', port=3000, label='node-odx-1') + + def tearDown(self): + with connection.cursor() as c: + c.execute('DROP DATABASE IF EXISTS {}'.format(self.target_name)) + + def assertImported(self): + self.assertEqual(psql(self.target, "SELECT name FROM app_project"), 'Imported project') + self.assertEqual(psql(self.target, "SELECT hostname || ' ' || label FROM nodeodm_processingnode"), 'node-odm-1 node-odm-1') + self.assertEqual(psql(self.target, "SELECT count(*) FROM auth_user WHERE username = 'webodm'"), '1') + + def test_import_into_empty_database(self): + import_database(self.source, self.target) + self.assertImported() + + def test_import_replaces_unused_database(self): + # A started but unused WebUI database: tables, boot rows, no user data + psql(self.target, "CREATE TABLE app_project (id int); CREATE TABLE app_task (id int); " + "CREATE TABLE auth_user (username text); INSERT INTO auth_user VALUES ('AnonymousUser'); " + "CREATE TABLE app_theme (id int); INSERT INTO app_theme VALUES (1)") + import_database(self.source, self.target) + self.assertImported() + + def test_import_refuses_database_with_user_data(self): + import_database(self.source, self.target) + with self.assertRaises(CommandError): + import_database(self.source, self.target) + self.assertImported() diff --git a/docker-compose.nodemicmac.yml b/docker-compose.nodemicmac.yml index bc978a95..0d3e10c3 100644 --- a/docker-compose.nodemicmac.yml +++ b/docker-compose.nodemicmac.yml @@ -11,7 +11,7 @@ services: - WO_CREATE_MICMAC_PNODE=YES node-micmac-1: image: opendronemap/nodemicmac - container_name: node-micmac-1 + container_name: webui-node-micmac-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.nodeodm.gpu.nvidia.yml b/docker-compose.nodeodm.gpu.nvidia.yml index c35ac7be..03dd2e0d 100644 --- a/docker-compose.nodeodm.gpu.nvidia.yml +++ b/docker-compose.nodeodm.gpu.nvidia.yml @@ -11,7 +11,7 @@ services: - WO_DEFAULT_NODES node-odm-1: image: opendronemap/nodeodm:3.6.2-gpu - container_name: node-odm-1 + container_name: webui-node-odm-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 79224466..a5a8a81c 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -11,7 +11,7 @@ services: - WO_DEFAULT_NODES node-odm-1: image: opendronemap/nodeodm:3.6.2 - container_name: node-odm-1 + container_name: webui-node-odm-1 expose: - "3000" restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 13b301d4..8e521e6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,12 @@ version: '2.1' volumes: dbdata: appmedia: + name: ${WO_MEDIA_VOLUME:-${COMPOSE_PROJECT_NAME}_appmedia} + external: ${WO_MEDIA_EXTERNAL:-false} services: db: image: postgis/postgis:18-3.6-alpine - container_name: db + container_name: webui-db expose: - "5432" volumes: @@ -30,7 +32,7 @@ services: oom_score_adj: -100 webapp: image: opendronemap/webui - container_name: webapp + container_name: webui-webapp entrypoint: /bin/bash -c "crond && chmod +x /webui/*.sh && /bin/bash -c \"/webui/wait-for-postgres.sh db /webui/wait-for-it.sh -t 0 broker:6379 -- /webui/start.sh\"" volumes: - ${WO_MEDIA_DIR}:/webui/app/media:z @@ -54,12 +56,12 @@ services: oom_score_adj: 0 broker: image: valkey/valkey:9.1.0-alpine3.23 - container_name: broker + container_name: webui-broker restart: unless-stopped oom_score_adj: -500 worker: image: opendronemap/webui - container_name: worker + container_name: webui-worker entrypoint: /bin/bash -c "/webui/wait-for-postgres.sh db /webui/wait-for-it.sh -t 0 broker:6379 -- /webui/wait-for-it.sh -t 0 webapp:8000 -- /webui/worker.sh start" volumes: - ${WO_MEDIA_DIR}:/webui/app/media:z diff --git a/import-webodm.sh b/import-webodm.sh new file mode 100644 index 00000000..703f4af5 --- /dev/null +++ b/import-webodm.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# Sourced by webui.sh. Imports the database of a WebODM installation into this one. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + echo "Run ./webui.sh import-webodm" + exit 1 +fi + +import_webodm_container="webui-import-webodm-db" +import_webodm_image="webodm/webodm_db" +import_webodm_new_volume="" + +# Git Bash rewrites arguments that look like POSIX paths (/data/PG_VERSION, +# /c/dir:/data) into Windows paths before docker sees them. MSYS_NO_PATHCONV +# turns that off for one command and is ignored everywhere else. +import_webodm_docker_run(){ + MSYS_NO_PATHCONV=1 docker run "$@" +} + +import_webodm_db_image(){ + $docker_compose -f docker-compose.yml config --images db 2> /dev/null +} + +# WebODM keeps a Postgres 9.5 cluster at the root of its database directory. +# Postgres 18 keeps its cluster in 18/docker, so a PG_VERSION file at the root +# can only be WebODM's. The directory belongs to the postgres user with mode +# 0700, so the check runs in a container. +import_webodm_is_webodm_dir(){ + [ -d "$1" ] && import_webodm_docker_run --rm -v "$1:/data:ro" --entrypoint test "$(import_webodm_db_image)" -f /data/PG_VERSION > /dev/null 2>&1 +} + +import_webodm_project(){ + echo "${COMPOSE_PROJECT_NAME:-$(basename "$__dirname" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_-]//g; s/^[-_]*//')}" +} + +import_webodm_dir_flags(){ + [[ $WO_DB_DIR = /* ]] && echo -n " --db-dir $WO_DB_DIR" + [[ $WO_MEDIA_DIR = /* ]] && echo -n " --media-dir $WO_MEDIA_DIR" + return 0 +} + +import_webodm_start_hint(){ + echo "Start OpenDroneMap WebUI with: $0 start$(import_webodm_dir_flags)" +} + +# Two Postgres servers on one data directory corrupt it +import_webodm_check_webodm_stopped(){ + if docker ps --format '{{.Image}}' | grep -q '^webodm/webodm_db'; then + echo -e "\033[91mWebODM is running. Stop it first with webodm.sh stop\033[39m" + exit 1 + fi +} + +# Called by start. A WebODM database in a docker volume is offered for import +# until this installation's own volume exists. Postgres 18 cannot use a WebODM +# database directory, so a start on one is refused with the import command. +import_webodm_offer(){ + local answer + if [[ $WO_DB_DIR = /* ]]; then + if import_webodm_is_webodm_dir "$WO_DB_DIR"; then + echo -e "\033[91mPostgres 18 cannot start in $WO_DB_DIR: it holds a WebODM database.\033[39m" + echo "To keep your WebODM projects, tasks and users, run: $0 import-webodm --webodm-db-dir $WO_DB_DIR --db-dir " + echo "Leave out --db-dir to import into a docker volume instead." + echo "To start with an empty installation, pass --db-dir pointing at an empty directory." + exit 1 + fi + return + fi + if ! docker volume inspect webodm_dbdata > /dev/null 2>&1 || docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then + return + fi + import_webodm_check_webodm_stopped + if [[ $skip_import = true ]]; then + return + fi + if [[ $detached = true ]] || [ ! -t 0 ]; then + echo -e "\033[91mFound a WebODM database (docker volume webodm_dbdata).\033[39m" + echo "To keep your WebODM projects, tasks and users, run: $0 import-webodm$(import_webodm_dir_flags)" + echo "To start with an empty installation instead, run: $0 start --skip-import" + exit 1 + fi + read -r -p "Found a WebODM database (docker volume webodm_dbdata). Import it? [Y/n] " answer + if [[ $answer =~ ^[Nn] ]]; then + echo "Starting with an empty installation." + else + import_webodm + fi +} + +# Run a command showing its output only if it fails +import_webodm_quiet(){ + local output + if ! output=$("$@" 2>&1); then + echo "$output" + return 1 + fi +} + +import_webodm_stop_servers(){ + docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true + $docker_compose -f docker-compose.yml rm -sf db > /dev/null 2>&1 || true +} + +# Exit trap while the import runs. The database volume this run created goes +# too, so the next start offers the import again. +import_webodm_failed(){ + trap - EXIT + import_webodm_stop_servers + if [ -n "$import_webodm_new_volume" ]; then + docker volume rm "$import_webodm_new_volume" > /dev/null 2>&1 || true + fi + echo -e "\033[91mImport failed. Your WebODM data is unchanged.\033[39m" + exit 1 +} + +# Copies the WebODM database (docker volume webodm_dbdata, or the directory +# given with --webodm-db-dir) into this installation's database (its docker +# volume, or the directory given with --db-dir). WebODM's media volume is +# reused in place. Nothing of WebODM's is modified. +import_webodm(){ + local source target network output imported media_volume + + if [ -n "$WO_WEBODM_DB_DIR" ]; then + source="$WO_WEBODM_DB_DIR" + if ! import_webodm_is_webodm_dir "$source"; then + echo -e "\033[91mNo WebODM database in $source.\033[39m" + exit 1 + fi + elif docker volume inspect webodm_dbdata > /dev/null 2>&1; then + source="webodm_dbdata" + else + echo -e "\033[91mNo WebODM database found (docker volume webodm_dbdata). For a WebODM started with --db-dir, pass --webodm-db-dir.\033[39m" + exit 1 + fi + + if [[ $WO_DB_DIR = /* ]]; then + if [ "$WO_DB_DIR" = "$source" ] || import_webodm_is_webodm_dir "$WO_DB_DIR"; then + echo -e "\033[91m$WO_DB_DIR holds a WebODM database. Pass --db-dir pointing at a new directory.\033[39m" + exit 1 + fi + target="$WO_DB_DIR" + else + target="docker volume $(import_webodm_project)_dbdata" + if ! docker volume inspect "$(import_webodm_project)_dbdata" > /dev/null 2>&1; then + import_webodm_new_volume="$(import_webodm_project)_dbdata" + fi + fi + import_webodm_check_webodm_stopped + + if [[ $WO_MEDIA_DIR != /* ]] && docker volume inspect webodm_appmedia > /dev/null 2>&1; then + media_volume="webodm_appmedia" + export WO_MEDIA_VOLUME="$media_volume" + export WO_MEDIA_EXTERNAL=true + fi + + echo "Importing from WebODM" + echo "================================" + echo "WebODM database: $source" + echo "OpenDroneMap WebUI database: $target" + if [ -n "$media_volume" ]; then + echo "WebODM media: docker volume $media_volume (used in place)" + fi + echo "================================" + + trap import_webodm_failed EXIT + docker rm -f "$import_webodm_container" > /dev/null 2>&1 || true + import_webodm_quiet $docker_compose -f docker-compose.yml up --no-start db + network=$(docker inspect webui-db --format '{{range $name, $_ := .NetworkSettings.Networks}}{{$name}}{{end}}') + + echo "Starting the WebODM database..." + import_webodm_quiet import_webodm_docker_run -d --name "$import_webodm_container" --network "$network" -v "$source:/var/lib/postgresql/data" "$import_webodm_image" + + echo "Starting the OpenDroneMap WebUI database..." + import_webodm_quiet $docker_compose -f docker-compose.yml up -d db + + echo "Copying the database..." + output=$($docker_compose -f docker-compose.yml run --rm --no-deps --entrypoint python webapp manage.py importwebodm --host "$import_webodm_container" 2>&1) && imported=true || imported=false + echo "$output" | grep -vE '^\s*$|pkg_resources|Generated secret|Container .* (Creating|Created)|mount of type' || true + if [[ $imported != true ]]; then + exit 1 + fi + + trap - EXIT + import_webodm_stop_servers + if [ -n "$media_volume" ] && ! grep -q '^WO_MEDIA_VOLUME=' "${__dirname}/.env"; then + printf 'WO_MEDIA_VOLUME=%s\nWO_MEDIA_EXTERNAL=true\n' "$media_volume" >> "${__dirname}/.env" + fi + echo -e "\033[1mDone!\033[0m WebODM data imported." +} diff --git a/nodeodm/migrations/0008_rename_default_odm_node.py b/nodeodm/migrations/0008_rename_default_odm_node.py index b66f4f1c..aa153252 100644 --- a/nodeodm/migrations/0008_rename_default_odm_node.py +++ b/nodeodm/migrations/0008_rename_default_odm_node.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals from django.db import migrations -from nodeodm.models import ProcessingNode def rename_default_node(apps, schema_editor): + ProcessingNode = apps.get_model('nodeodm', 'ProcessingNode') for default_node in ProcessingNode.objects.filter(hostname='node-odm-1'): default_node.hostname = 'webui_node-odm_1' default_node.label = 'node-odm-1' diff --git a/nodeodm/migrations/0010_rename_odm_to_odx.py b/nodeodm/migrations/0010_rename_odm_to_odx.py index d00d9a38..78bd052b 100644 --- a/nodeodm/migrations/0010_rename_odm_to_odx.py +++ b/nodeodm/migrations/0010_rename_odm_to_odx.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from django.db import migrations -from nodeodm.models import ProcessingNode from django.db.models import Q def rename_default_node(apps, schema_editor): + ProcessingNode = apps.get_model('nodeodm', 'ProcessingNode') for default_node in ProcessingNode.objects.filter(Q(hostname='node-odm-1') | Q(label='node-odm-1')): default_node.hostname = 'node-odm-1' default_node.label = 'node-odm-1' diff --git a/webui.sh b/webui.sh index ea01700a..369b90ad 100755 --- a/webui.sh +++ b/webui.sh @@ -30,6 +30,7 @@ fi # Load default values source "${__dirname}/.env" +source "${__dirname}/import-webodm.sh" DEFAULT_PORT="$WO_PORT" DEFAULT_HOST="$WO_HOST" DEFAULT_MEDIA_DIR="$WO_MEDIA_DIR" @@ -161,6 +162,16 @@ case $key in export WO_IPV6=YES shift # past argument ;; + --skip-import) + skip_import=true + shift # past argument + ;; + --webodm-db-dir) + WO_WEBODM_DB_DIR=$(realpath "$2") + export WO_WEBODM_DB_DIR + shift # past argument + shift # past value + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -189,6 +200,7 @@ usage(){ echo " rebuild Rebuild all docker containers and perform cleanups" echo " checkenv Do an environment check and install missing components" echo " test [frontend|backend] [args] Run tests (all tests, or just frontend/backend with optional arguments)" + echo " import-webodm [options] Import the database of a stopped WebODM installation into this unused OpenDroneMap WebUI installation and use its media in place. Pass --webodm-db-dir for a WebODM started with --db-dir." echo " resetadminpassword \"\" Reset the administrator's password to a new one. OpenDroneMap WebUI must be running when executing this command and the password must be enclosed in double quotes." echo "" echo "Options:" @@ -213,6 +225,8 @@ usage(){ echo " --worker-memory Maximum amount of memory allocated for the worker process (default: unlimited)" echo " --worker-cpus Maximum number of CPUs allocated for the worker process (default: all)" echo " --ipv6 Enable IPV6" + echo " --skip-import Start with an empty installation even if a WebODM database is found (default: ask)" + echo " --webodm-db-dir Database directory of a WebODM installation started with --db-dir, for import-webodm (default: docker volume webodm_dbdata)" exit } @@ -436,6 +450,8 @@ start(){ # mkdir -p $WO_MEDIA_DIR # mkdir -p webui/app/media/tmp + import_webodm_offer + command="$docker_compose -f docker-compose.yml" if [[ $WO_DEFAULT_NODES -gt 0 ]]; then @@ -581,7 +597,7 @@ resetpassword(){ newpass=$1 if [[ -n "$newpass" ]]; then - container_hash=$(docker ps -q --filter "name=webapp") + container_hash=$(docker ps -q --filter "name=^webui-webapp$") if [[ -z "$container_hash" ]]; then echo -e "\033[91mCannot find webapp docker container. Is OpenDroneMap WebUI running?\033[39m" exit 1 @@ -676,6 +692,10 @@ elif [[ $1 = "checkenv" ]]; then elif [[ $1 = "test" ]]; then shift || true run_tests "$@" +elif [[ $1 = "import-webodm" ]]; then + environment_check + import_webodm + import_webodm_start_hint elif [[ $1 = "resetadminpassword" ]]; then resetpassword "$2" else