diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 309dc8965a..a8276c4876 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,9 @@ jobs: tests: name: Backend tests runs-on: ubuntu-latest - timeout-minutes: 45 + # The plugin e2e modules alone need well over half an hour: every scheduler-based + # engine (presto, trino) starts its own container per documented version. + timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -97,8 +99,19 @@ jobs: # The root pom has no Surefire pin; Maven's default (2.12.4) cannot run # JUnit 5 and silently executes zero tests. Invoke Surefire explicitly # until the root pom pins the plugin version. + # + # Install first (skipping test execution) so downstream modules resolve + # packaged artifacts: the shaded drivers only exist after the package + # phase, and a bare test-compile would leave their plugin tests with a + # driver-less classpath. + - name: Build and install all modules + run: > + ./mvnw -B clean install + -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip + -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm + - name: Run the JUnit 5 suite run: > - ./mvnw -B test-compile org.apache.maven.plugins:maven-surefire-plugin:3.2.2:test + ./mvnw -B org.apache.maven.plugins:maven-surefire-plugin:3.2.2:test -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e52acaddc..79282ec7cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,46 +109,133 @@ jobs: maven: name: Publish to Maven Central runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 150 needs: build + env: + # Publishing to Maven Central is permanent, so it runs last and only when + # its secrets exist; a half-configured repository still releases the rest. + CENTRAL_CONFIGURED: ${{ secrets.CENTRAL_USERNAME != '' && secrets.GPG_PRIVATE_KEY != '' }} steps: - uses: actions/checkout@v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }} - # Imports the GPG key and injects the metaforge-ossrh server - # credentials into the Maven settings. The gpg-passphrase input is the - # NAME of the env variable holding the passphrase, provided below. - - name: Set up JDK 21 with signing and repository credentials + # The distribution and the store archives (dist/plugins) are built once + # here; the publishing steps below reuse them, so a deploy that fails + # half-way leaves nothing behind and re-running finishes the release. + - name: Set up JDK 21 uses: actions/setup-java@v4 with: distribution: temurin java-version: 21 cache: maven - gpg-private-key: ${{ secrets.GPG_SECRET }} - gpg-passphrase: GPG_PASSPHRASE - server-id: metaforge-ossrh - server-username: OSSRH_USERNAME - server-password: OSSRH_TOKEN - - # Fails fast with a clear message when the key or passphrase is wrong, - # instead of stalling or failing halfway through the deploy. - - name: Verify the GPG signing key + - name: Build the distribution and the store archives + run: > + ./mvnw -B clean install + -Dmaven.test.skip=true -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip + -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm + + - uses: actions/setup-java@v4 + if: env.CENTRAL_CONFIGURED == 'true' + with: + distribution: temurin + java-version: 21 + cache: maven + # Publishing goes to Maven Central through the Central Portal with the + # server id "central"; the gpg-passphrase input is the NAME of the env + # variable holding the passphrase, provided below. + server-id: central + server-username: CENTRAL_USERNAME + server-password: CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Publish the Maven artifacts to Maven Central + if: env.CENTRAL_CONFIGURED == 'true' env: - GPG_PASSPHRASE: ${{ secrets.GPG_PASSWORD }} + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | - gpg --list-secret-keys --keyid-format long - echo 'metaforge signing check' | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --local-user "$(gpg --list-secret-keys --keyid-format long --with-colons | awk -F: '/^sec/{print $5; exit}')" -o /tmp/signing-check.sig --sign - rm -f /tmp/signing-check.sig + version="$(./mvnw -B -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive \ + -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN exec:exec | tail -1)" + if curl --silent --fail --head --output /dev/null \ + "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge/${version}/metaforge-${version}.pom"; then + echo "::notice::${version} is on Maven Central already" + exit 0 + fi + ./mvnw --batch-mode --no-transfer-progress -Prelease,central -Dcentral.skip=false \ + -Dmaven.test.skip=true -Dcheckstyle.skip -Dspotbugs.skip \ + -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm deploy \ + -pl '!test/metaforge-test-condor,!test/metaforge-test-convert,!test/metaforge-test-core,!test/metaforge-test-lib,!test/metaforge-test-executor,!test/metaforge-test-scheduler,!test/metaforge-test-fs,!test/metaforge-test-driver,!test/metaforge-test-parser,!test/metaforge-test-plugin' + - name: Skip Maven Central + if: env.CENTRAL_CONFIGURED != 'true' + run: echo "::notice::Maven Central skipped; set CENTRAL_USERNAME, CENTRAL_PASSWORD, GPG_PRIVATE_KEY and GPG_PASSPHRASE to publish there" + + # The store catalogue and the statistics catalog are built from the very + # archives just deployed to Maven Central (dist/plugins holds copies), so + # one SHA-256 and size are valid for the Maven and CDN sources alike. + - name: Build the store catalogue and the statistics catalog + run: > + node configure/publish/build-metadata.mjs --dist dist/plugins --strict + --out dist/metadata.json --catalog dist/release-catalog.json + --commit "$GITHUB_SHA" --generation "$GITHUB_RUN_NUMBER" --released now + + - name: Upload the catalogues + uses: actions/upload-artifact@v4 + with: + name: store-catalogue + if-no-files-found: error + path: | + dist/metadata.json + dist/release-catalog.json - - name: Deploy to Maven Central + # Order: packages to the CDN -> catalog into ForgeStats -> public metadata.json. + # A failed step keeps the previous public catalogue, and re-running is safe. + - name: Publish the packages to the CDN env: - OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} - OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSWORD }} - run: > - ./mvnw -B clean deploy -Pdeploy - -DskipTests -Dcheckstyle.skip -Dspotbugs.skip - -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm - -Dgpg.passphrase="$GPG_PASSPHRASE" - -pl '!test/metaforge-test-condor,!test/metaforge-test-convert,!test/metaforge-test-core,!test/metaforge-test-lib,!test/metaforge-test-executor,!test/metaforge-test-scheduler,!test/metaforge-test-fs,!test/metaforge-test-driver,!test/metaforge-test-parser,!test/metaforge-test-plugin' + OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} + OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET }} + run: | + if [ -z "$OSS_ACCESS_KEY_ID" ]; then + echo '::warning::OSS secrets are not set; skipping the CDN upload (the store falls back to Maven Central)' + exit 0 + fi + # Official ossutil 1.x installer (same tool as configure/publish/publish-oss-artifact.sh). + curl -fsSL https://gosspublic.alicdn.com/ossutil/install.sh | sudo bash + oss() { ossutil -e "$OSS_ENDPOINT" -i "$OSS_ACCESS_KEY_ID" -k "$OSS_ACCESS_KEY_SECRET" "$@"; } + jq -r '.plugins[] | .sources.cdn' dist/release-catalog.json | while read -r path; do + file="dist/plugins/$(basename "$path")" + oss cp "$file" "oss://$OSS_BUCKET/applications/metaforge/plugins/$path" -f + done + + - name: Import the statistics catalog into ForgeStats + env: + STATS_SSH_KEY: ${{ secrets.STATS_SSH_KEY }} + STATS_SSH_TARGET: ${{ secrets.STATS_SSH_TARGET }} + run: | + if [ -z "$STATS_SSH_KEY" ] || [ -z "$STATS_SSH_TARGET" ]; then + echo '::warning::ForgeStats secrets are not set; downloads of this release are not counted' + exit 0 + fi + umask 077 + printf '%s\n' "$STATS_SSH_KEY" > "$RUNNER_TEMP/stats-key" + digest=$(sha256sum dist/release-catalog.json | cut -d ' ' -f 1) + ssh -T -i "$RUNNER_TEMP/stats-key" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new \ + "$STATS_SSH_TARGET" "sudo -n /usr/local/sbin/forgestats-admin catalog metaforge $digest" < dist/release-catalog.json + rm -f "$RUNNER_TEMP/stats-key" + + - name: Publish the store catalogue + env: + OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} + OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} + OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }} + OSS_BUCKET: ${{ secrets.OSS_BUCKET }} + run: | + if [ -z "$OSS_ACCESS_KEY_ID" ]; then + echo '::warning::OSS secrets are not set; publish dist/metadata.json from the store-catalogue artifact manually' + exit 0 + fi + ossutil -e "$OSS_ENDPOINT" -i "$OSS_ACCESS_KEY_ID" -k "$OSS_ACCESS_KEY_SECRET" \ + cp dist/metadata.json "oss://$OSS_BUCKET/applications/metaforge/metadata/metadata.json" -f diff --git a/README.md b/README.md index d4d1307594..e2a4ce84d3 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,14 @@ # MetaForge ---- +**Integrated software for data transformation, integration, and visualization** -![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fmetaforge.git&countColor=%23263759&style=flat&labelStyle=none) -[![](https://tokei.rs/b1/github/EdurtIO/metaforge)](https://github.com/EdurtIO/metaforge) -![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) +--- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -![GitHub Release Date](https://img.shields.io/github/release-date/EdurtIO/metaforge?style=flat-square) +![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) -![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge) +[![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge)](https://hub.docker.com/r/devliveorg/metaforge) ![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/devliveorg/metaforge?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/devliveorg/metaforge?style=flat-square) @@ -23,28 +21,118 @@ -### What is metaforge? +## What is MetaForge? + +--- + +MetaForge is integrated software for data transformation, integration, and visualization. It supports a wide variety of data sources — relational databases, NoSQL databases, big data engines, message queues and file systems — and lets you manage multiple data sources, query and transform data, build datasets, and create charts and monitoring views. + +## Features --- -MetaForge is integrated software for data transformation, integration, and visualization. Support a variety of data sources, file types, big data related database, relational database, NoSQL database, etc. Through the software can realize the management of multiple data sources, the data under the source of various operations conversion, making data charts, monitoring data sources and other functions. +- **Data source management** — connect, test and manage 50+ data sources from a single console +- **SQL query console** — IDE-style editor with syntax highlighting, formatting, execution history and export +- **Datasets** — build reusable datasets from tables or ad-hoc queries, preview data and define column metadata +- **Visualization** — drag-and-drop dashboards with tables, pies and more chart types +- **Reports** — schedule and share reports +- **Workflows** — chain data jobs into repeatable workflows +- **File storage** — read and write files through Aliyun OSS, Amazon S3, HDFS, local storage and more +- **Notifications** — DingTalk, email and other channels for job and system events +- **AI assistant** — chat with your data through configured AI providers +- **Plugin architecture** — every connector, converter and storage backend is a hot-loadable plugin +- **Administration** — users, roles, menus and runtime configuration with fine-grained permissions + +## Requirements -| Username | Password | -|:---------:|:------------| -| `metaforge` | `123456789` | -| `admin` | `12345678` | +--- + +| Dependency | Version | +|:-----------|:---------------------------------------------| +| JDK | 21 | +| Maven | 3.9+ (wrapper `./mvnw` included) | +| Node.js | 20 | +| pnpm | 9.13.2 | -### Require +## Getting Started --- -> Must-read for users: Be sure to execute the following command after cloning the code locally +> Must-read for users: be sure to execute the following command after cloning the code locally ```bash cp configure/git-hook/* .git/hooks chmod 700 .git/hooks/* ``` +### Run the backend (development) + +Run the main class `org.devlive.metaforge.server.MetaForge` (module `core/metaforge-server`) from your IDE. +The server starts on [http://localhost:9096](http://localhost:9096). On first launch the setup wizard guides you through +selecting the metadata database (embedded H2 by default, or MySQL / PostgreSQL / SQL Server / Oracle) and creating the +administrator account. + +### Run the frontend (development) + +```bash +pnpm install +pnpm --filter metaforge-web dev +``` + +The web console starts on [http://localhost:3001](http://localhost:3001) and proxies API calls to the backend. + +### Build for production + +```bash +# Frontend +pnpm --filter metaforge-web build + +# Backend + distribution package (the server serves the built frontend +# from core/metaforge-web/dist; the tarball lands in dist/) +./mvnw -T1C clean package -DskipTests +``` + +## Docker + +--- + +```bash +docker pull devliveorg/metaforge:latest +docker run -d --name metaforge -p 9096:9096 devliveorg/metaforge:latest +``` + +Or unpack the distribution built locally (`dist/metaforge-server-2026.0.0-bin.tar.gz`) and run `bin/startup.sh`. + +## Project Structure + +--- + +``` +metaforge +├── client # Command line client +├── configure # Default configuration and plugin manifests +├── core +│ ├── metaforge-captcha # Captcha generation +│ ├── metaforge-common # Shared utilities and enums +│ ├── metaforge-condor # Scheduling integration +│ ├── metaforge-server # Spring Boot server (REST API entry point) +│ ├── metaforge-service # Domain services and persistence +│ ├── metaforge-security # Security configuration +│ ├── metaforge-sql # SQL statement builders +│ ├── metaforge-spi # Plugin SPI contracts +│ ├── metaforge-web # Vue 3 + Tailwind web console +├── convert # Data converters (JSON, CSV, XML, ...) +├── driver # Database drivers (Redis, MongoDB, ...) +├── executor # Execution engines +├── fs # File storage plugins (OSS, S3, local, ...) +├── lib # Shared libraries (HTTP client, logging, shell) +├── notify # Notification channels (DingTalk, ...) +├── parser # SQL parsers (MySQL, Trino, ...) +├── plugin # Data source plugins (50+ connectors) +├── scheduler # Job scheduling +└── shaded # Shaded third-party integrations +``` + ## Supported Connectors --- @@ -105,17 +193,11 @@ Here are some of the major database solutions that are supported: DolphinDB -## System architecture - ---- - - - -## Stargazers over time +## Documentation --- -[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) +[Extended documentation for MetaForge](https://metaforge.devlive.org) ## Join Us @@ -131,11 +213,11 @@ Please visit [Issues 950](https://github.com/devlive-community/metaforge/issues/ [![App Store](https://img.shields.io/badge/App%20Store-Rainbond-brightgreen?style=flat-square)](https://www.rainbond.com/) [![Vue 3](https://img.shields.io/badge/UI-Vue%203%20%2B%20Tailwind-brightgreen?style=flat-square)](https://vuejs.org/) -## Installation and Configuration +## Stargazers over time --- -[Extended documentation for MetaForge](https://metaforge.devlive.org) +[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) ## Contributors diff --git a/README.ru.md b/README.ru.md index de30f8eca8..525539e4da 100644 --- a/README.ru.md +++ b/README.ru.md @@ -4,16 +4,14 @@ # MetaForge ---- +**Интегрированное программное обеспечение для преобразования, интеграции и визуализации данных** -![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fmetaforge.git&countColor=%23263759&style=flat&labelStyle=none) -[![](https://tokei.rs/b1/github/EdurtIO/metaforge)](https://github.com/EdurtIO/metaforge) -![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) +--- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -![GitHub Release Date](https://img.shields.io/github/release-date/EdurtIO/metaforge?style=flat-square) +![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) -![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge) +[![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge)](https://hub.docker.com/r/devliveorg/metaforge) ![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/devliveorg/metaforge?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/devliveorg/metaforge?style=flat-square) @@ -23,35 +21,124 @@ -### Что такое MetaForge? +## Что такое MetaForge? --- -MetaForge — это интегрированное программное обеспечение для преобразования, интеграции и визуализации данных. Поддержка нескольких источников данных, типов файлов, баз данных, связанных с большими данными, реляционных баз данных, баз данных NoSQL и многого другого. С помощью программного обеспечения может быть реализовано управление несколькими источниками данных, а также могут выполняться различные операции с данными в источнике, могут быть построены диаграммы данных, могут быть обнаружены источники данных и могут быть выполнены другие функции. +MetaForge — это интегрированное программное обеспечение для преобразования, интеграции и визуализации данных. Оно поддерживает самые разные источники данных — реляционные СУБД, NoSQL-базы данных, big data движки, очереди сообщений и файловые системы — и позволяет управлять множеством источников из единой консоли, запрашивать и преобразовывать данные, строить наборы данных, диаграммы и представления мониторинга. + +## Возможности + +--- + +- **Управление источниками данных** — подключение, проверка и управление 50+ источниками из единой консоли +- **SQL-консоль** — редактор в стиле IDE с подсветкой синтаксиса, форматированием, историей выполнения и экспортом +- **Наборы данных** — создание переиспользуемых наборов из таблиц или произвольных запросов, предпросмотр данных и метаданные столбцов +- **Визуализация** — дашборды с перетаскиванием: таблицы, круговые диаграммы и другие типы графиков +- **Отчёты** — планирование и совместный доступ к отчётам +- **Рабочие процессы** — объединение задач обработки данных в повторяемые workflow +- **Файловое хранилище** — чтение и запись файлов через Aliyun OSS, Amazon S3, HDFS, локальное хранилище и другие +- **Уведомления** — DingTalk, электронная почта и другие каналы для событий задач и системы +- **AI-ассистент** — общайтесь со своими данными через настроенных AI-провайдеров +- **Плагинная архитектура** — каждый коннектор, конвертер и хранилище являются подключаемыми плагинами +- **Администрирование** — пользователи, роли, меню и конфигурация среды выполнения с тонкой настройкой прав + +## Требования + +--- -| Имя пользователя | пароль | -|:----------------:|:------------| -| `metaforge` | `123456789` | -| `admin` | `12345678` | +| Зависимость | Версия | +|:------------|:----------------------------------| +| JDK | 21 | +| Maven | 3.9+ (в комплекте `./mvnw`) | +| Node.js | 20 | +| pnpm | 9.13.2 | -### просьба +## Быстрый старт --- -> Обязательный к прочтению пользователь: обязательно клонируйте код локально и выполните следующую команду +> Обязательно к прочтению: после клонирования кода обязательно выполните следующие команды ```bash cp configure/git-hook/* .git/hooks chmod 700 .git/hooks/* ``` +### Запуск бэкенда (разработка) + +Запустите главный класс `org.devlive.metaforge.server.MetaForge` (модуль `core/metaforge-server`) в своей IDE. +Сервер доступен на [http://localhost:9096](http://localhost:9096). При первом запуске мастер установки поможет выбрать +базу метаданных (по умолчанию встроенная H2, также MySQL / PostgreSQL / SQL Server / Oracle) и создать +учётную запись администратора. + +### Запуск фронтенда (разработка) + +```bash +pnpm install +pnpm --filter metaforge-web dev +``` + +Веб-консоль доступна на [http://localhost:3001](http://localhost:3001) и проксирует API-запросы к бэкенду. + +### Сборка для продакшена + +```bash +# Фронтенд +pnpm --filter metaforge-web build + +# Бэкенд + дистрибутив (сервер отдаёт собранный фронтенд +# из core/metaforge-web/dist; архив попадает в dist/) +./mvnw -T1C clean package -DskipTests +``` + +## Docker + +--- + +```bash +docker pull devliveorg/metaforge:latest +docker run -d --name metaforge -p 9096:9096 devliveorg/metaforge:latest +``` + +Либо распакуйте собранный локально дистрибутив (`dist/metaforge-server-2026.0.0-bin.tar.gz`) и запустите `bin/startup.sh`. + +## Структура проекта + +--- + +``` +metaforge +├── client # Консольный клиент +├── configure # Конфигурация по умолчанию и манифесты плагинов +├── core +│ ├── metaforge-captcha # Генерация капчи +│ ├── metaforge-common # Общие утилиты и перечисления +│ ├── metaforge-condor # Интеграция планировщика +│ ├── metaforge-server # Сервер Spring Boot (точка входа REST API) +│ ├── metaforge-service # Доменные сервисы и хранение данных +│ ├── metaforge-security # Конфигурация безопасности +│ ├── metaforge-sql # Конструкторы SQL-запросов +│ ├── metaforge-spi # Контракты plugin SPI +│ ├── metaforge-web # Веб-консоль Vue 3 + Tailwind +├── convert # Конвертеры данных (JSON, CSV, XML, ...) +├── driver # Драйверы баз данных (Redis, MongoDB, ...) +├── executor # Движки выполнения +├── fs # Плагины файловых хранилищ (OSS, S3, локальное, ...) +├── lib # Общие библиотеки (HTTP-клиент, логирование, shell) +├── notify # Каналы уведомлений (DingTalk, ...) +├── parser # SQL-парсеры (MySQL, Trino, ...) +├── plugin # Плагины источников данных (50+ коннекторов) +└── shaded # Shaded сторонние интеграции +``` + ## Поддерживаемые коннекторы --- -MetaForge может запрашивать данные из любого хранилища данных или движка данных (ClickHouse, MySQL, Presto и т.д.), использующего SQL. +MetaForge может запрашивать данные из любого SQL-хранилища или движка данных (ClickHouse, MySQL, Presto и др.). -Ниже приведены некоторые из основных поддерживаемых решений для баз данных. +Ниже представлены некоторые из основных поддерживаемых решений для баз данных:
ClickHouse @@ -105,25 +192,19 @@ MetaForge может запрашивать данные из любого хр DolphinDB
-## Архитектура системы +## Документация --- - - -## Атлас вкладов - ---- - -[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) +[Расширенная документация по MetaForge](https://metaforge.devlive.org) ## Присоединяйтесь к нам --- -Вид [Issues 950](https://github.com/devlive-community/metaforge/issues/950) +Посетите [Issues 950](https://github.com/devlive-community/metaforge/issues/950) -## Признание +## Благодарности --- @@ -131,13 +212,13 @@ MetaForge может запрашивать данные из любого хр [![App Store](https://img.shields.io/badge/App%20Store-Rainbond-brightgreen?style=flat-square)](https://www.rainbond.com/) [![Vue 3](https://img.shields.io/badge/UI-Vue%203%20%2B%20Tailwind-brightgreen?style=flat-square)](https://vuejs.org/) -## Установка и настройка +## История звёзд --- -[Расширенная документация по MetaForge](https://metaforge.devlive.org) +[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) -## Участников +## Участники --- diff --git a/README.zh_CN.md b/README.zh_CN.md index 9f4c79574b..993dffe945 100644 --- a/README.zh_CN.md +++ b/README.zh_CN.md @@ -4,16 +4,14 @@ # MetaForge ---- +**用于数据转换、集成和可视化的集成软件** -![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fmetaforge.git&countColor=%23263759&style=flat&labelStyle=none) -[![](https://tokei.rs/b1/github/EdurtIO/metaforge)](https://github.com/EdurtIO/metaforge) -![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) +--- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -![GitHub Release Date](https://img.shields.io/github/release-date/EdurtIO/metaforge?style=flat-square) +![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg) -![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge) +[![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge)](https://hub.docker.com/r/devliveorg/metaforge) ![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/devliveorg/metaforge?style=flat-square) ![Docker Pulls](https://img.shields.io/docker/pulls/devliveorg/metaforge?style=flat-square) @@ -23,18 +21,40 @@ -### 什么是 metaforge? +## 什么是 MetaForge? --- -MetaForge 是用于数据转换、集成和可视化的集成软件。支持多种数据源、文件类型、大数据相关数据库、关系数据库、NoSQL数据库等。通过软件可以实现多数据源的管理,对源下的数据进行各种操作转换、制作数据图表、监控数据源和其他功能。 +MetaForge 是用于数据转换、集成和可视化的集成软件。支持多种数据源 —— 关系型数据库、NoSQL 数据库、大数据引擎、消息队列和文件系统 —— 让您在一个控制台中管理多个数据源,查询和转换数据、构建数据集,并创建图表和监控视图。 + +## 功能特性 + +--- + +- **数据源管理** — 在单一控制台中连接、测试和管理 50+ 数据源 +- **SQL 查询控制台** — IDE 风格的编辑器,支持语法高亮、格式化、执行历史和导出 +- **数据集** — 基于表或临时查询构建可复用的数据集,预览数据并定义列元数据 +- **可视化** — 拖拽式仪表盘,支持表格、饼图等多种图表类型 +- **报表** — 定时调度并分享报表 +- **工作流** — 将数据任务串联为可重复执行的工作流 +- **文件存储** — 通过阿里云 OSS、Amazon S3、HDFS、本地存储等读写文件 +- **通知** — 钉钉、邮件等渠道的任务与系统事件通知 +- **AI 助手** — 通过配置的 AI 提供商与您的数据对话 +- **插件架构** — 每个连接器、转换器和存储后端都是可热加载的插件 +- **系统管理** — 用户、角色、菜单与运行时配置,支持细粒度权限 + +## 环境要求 + +--- -| 用户名 | 密码 | -|:---------:|:------------| -| `metaforge` | `123456789` | -| `admin` | `12345678` | +| 依赖 | 版本 | +|:---------|:------------------------------| +| JDK | 21 | +| Maven | 3.9+(已内置 `./mvnw`) | +| Node.js | 20 | +| pnpm | 9.13.2 | -### 要求 +## 快速开始 --- @@ -45,6 +65,72 @@ cp configure/git-hook/* .git/hooks chmod 700 .git/hooks/* ``` +### 运行后端(开发) + +在 IDE 中运行主类 `org.devlive.metaforge.server.MetaForge`(模块 `core/metaforge-server`)。 +服务启动后访问 [http://localhost:9096](http://localhost:9096)。首次启动时,安装向导会引导您选择元数据库 +(默认内嵌 H2,也可选择 MySQL / PostgreSQL / SQL Server / Oracle)并创建管理员账号。 + +### 运行前端(开发) + +```bash +pnpm install +pnpm --filter metaforge-web dev +``` + +Web 控制台启动在 [http://localhost:3001](http://localhost:3001),API 请求会代理到后端。 + +### 生产构建 + +```bash +# 前端 +pnpm --filter metaforge-web build + +# 后端 + 发行包(服务端会直接加载 core/metaforge-web/dist +# 下构建好的前端;压缩包输出到 dist/) +./mvnw -T1C clean package -DskipTests +``` + +## Docker + +--- + +```bash +docker pull devliveorg/metaforge:latest +docker run -d --name metaforge -p 9096:9096 devliveorg/metaforge:latest +``` + +或者解压本地构建的发行包(`dist/metaforge-server-2026.0.0-bin.tar.gz`),运行 `bin/startup.sh`。 + +## 项目结构 + +--- + +``` +metaforge +├── client # 命令行客户端 +├── configure # 默认配置与插件清单 +├── core +│ ├── metaforge-captcha # 验证码生成 +│ ├── metaforge-common # 通用工具与枚举 +│ ├── metaforge-condor # 调度集成 +│ ├── metaforge-server # Spring Boot 服务端(REST API 入口) +│ ├── metaforge-service # 领域服务与持久化 +│ ├── metaforge-security # 安全配置 +│ ├── metaforge-sql # SQL 语句构建器 +│ ├── metaforge-spi # 插件 SPI 契约 +│ ├── metaforge-web # Vue 3 + Tailwind Web 控制台 +├── convert # 数据转换器(JSON、CSV、XML 等) +├── driver # 数据库驱动(Redis、MongoDB 等) +├── executor # 执行引擎 +├── fs # 文件存储插件(OSS、S3、本地存储等) +├── lib # 共享库(HTTP 客户端、日志、Shell) +├── notify # 通知渠道(钉钉等) +├── parser # SQL 解析器(MySQL、Trino 等) +├── plugin # 数据源插件(50+ 连接器) +└── shaded # 第三方集成 Shade 封装 +``` + ## 支持的连接器 --- @@ -105,17 +191,11 @@ MetaForge 可以从任何使用 SQL 的数据存储或数据引擎(ClickHouse DolphinDB -## 系统架构 +## 文档 --- - - -## 贡献图谱 - ---- - -[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) +[MetaForge 的扩展文档](https://metaforge.devlive.org) ## 加入我们 @@ -131,11 +211,11 @@ MetaForge 可以从任何使用 SQL 的数据存储或数据引擎(ClickHouse [![App Store](https://img.shields.io/badge/App%20Store-Rainbond-brightgreen?style=flat-square)](https://www.rainbond.com/) [![Vue 3](https://img.shields.io/badge/UI-Vue%203%20%2B%20Tailwind-brightgreen?style=flat-square)](https://vuejs.org/) -## 安装和配置 +## Star 历史 --- -[MetaForge 的扩展文档](https://metaforge.devlive.org) +[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline) ## 贡献者 diff --git a/configure/docker/application.properties b/configure/docker/application.properties index f4e51ed0a4..063f19b4e5 100644 --- a/configure/docker/application.properties +++ b/configure/docker/application.properties @@ -96,6 +96,7 @@ metaforge.parser.sql.defaultEngine=Trino ################################# File System configure ################################# # For the file storage type, please refer to metaforge-fs-, which defaults to Local +# These are the defaults: an administrator sets file storage in System settings, which then takes precedence. # ------ Local File System ------ # metaforge.fs.type=LocalFs metaforge.fs.access= diff --git a/configure/etc/conf/application.properties b/configure/etc/conf/application.properties index fba5a1c21d..f25c684357 100644 --- a/configure/etc/conf/application.properties +++ b/configure/etc/conf/application.properties @@ -26,6 +26,8 @@ spring.web.resources.add-mappings=true spring.datasource.url=jdbc:mysql://localhost:3306/metaforge?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&useSSL=false&useOldAliasMetadataBehavior=true&jdbcCompliantTruncation=false&allowPublicKeyRetrieval=true spring.datasource.username=root spring.datasource.password=12345678 +# The metadata schema is migrated by MetaForge itself (DatabaseMigrator), never by Boot. +spring.liquibase.enabled=false ################################ Cache configure ################################# # Set redis environment @@ -96,6 +98,7 @@ metaforge.parser.sql.defaultEngine=Trino ################################# File System configure ################################# # For the file storage type, please refer to metaforge-fs-, which defaults to Local +# These are the defaults: an administrator sets file storage in System settings, which then takes precedence. # ------ Local File System ------ # metaforge.fs.type=LocalFs metaforge.fs.access= @@ -145,6 +148,14 @@ plugin.manager.extend.packages=com.fasterxml.jackson # 插件加载线程数:0 根据 CPU 自动选择(最多 4),1 串行;重启后生效 # Plugin loading workers: 0 = automatic (up to 4 CPUs), 1 = sequential; restart required plugin.manager.load-threads=0 +# 插件商店:经 ForgeStats 统计服务下载并统计下载量;留空 stats-url 则直接下载、不计数 +# Plugin store: downloads go through the ForgeStats statistics service; an empty stats-url downloads directly without counting +plugin.store.stats-url=https://stats.devlive.org +plugin.store.app=metaforge +# 下载来源:maven(Maven Central)、cdn(MetaForge CDN)或 mirror(自建镜像,目录结构同 CDN:{version}/{type}/{key}-bin.tar.gz) +# Download source: maven (Maven Central), cdn (MetaForge CDN) or mirror (self-hosted, CDN layout {version}/{type}/{key}-bin.tar.gz) +plugin.store.source=maven +plugin.store.mirror= ################################# Experimental features ################################# # This configuration is used to dynamically increase the total number of rows of returned data in SQL during query, and currently only takes effect for user-directed queries diff --git a/configure/etc/conf/executor/category.yaml b/configure/etc/conf/executor/category.yaml deleted file mode 100644 index 48984be673..0000000000 --- a/configure/etc/conf/executor/category.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- label: Input - value: source - supportExecutors: - - Seatunnel -- label: Transform - value: transform - supportExecutors: - - Seatunnel -- label: Output - value: sink - supportExecutors: - - Seatunnel diff --git a/configure/etc/conf/executor/sql-transform.yaml b/configure/etc/conf/executor/sql-transform.yaml deleted file mode 100644 index d8afe66940..0000000000 --- a/configure/etc/conf/executor/sql-transform.yaml +++ /dev/null @@ -1,36 +0,0 @@ -id: sql-transform -label: Sql -tid: sql-transform -key: Sql -description: SQL transform -category: transform -configure: - - field: source_table_name - label: Source table name - type: input - placeholder: Please input source table name - required: true - description: The source table name, the query SQL table name must match this field. - - field: result_table_name - label: Result table name - type: input - placeholder: Please input result table name - required: true - - field: query - label: Query - type: textarea - placeholder: Please input query - required: true - description: | - The query SQL, it's a simple SQL supported base function and criteria filter operation. But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like. -ports: - - id: input - label: Input - type: input - required: true - message: Input is required - - id: output - label: Output - type: output - required: true - message: Output is required diff --git a/configure/etc/conf/jvm.conf b/configure/etc/conf/jvm.conf index b3fc38b233..d42c5111e9 100644 --- a/configure/etc/conf/jvm.conf +++ b/configure/etc/conf/jvm.conf @@ -1,3 +1,4 @@ -server -Xmx2048m -Xms1024m +--add-opens=java.base/java.nio=ALL-UNNAMED diff --git a/configure/etc/conf/plugin.conf b/configure/etc/conf/plugin.conf index 7ca6327e65..8fdf666e6c 100644 --- a/configure/etc/conf/plugin.conf +++ b/configure/etc/conf/plugin.conf @@ -70,7 +70,7 @@ metaforge-fs-amazon-s3 metaforge-fs-minio -- Executor list -- -metaforge-executor-local +metaforge-executor-metaforge metaforge-executor-seatunnel -- Convert list -- diff --git a/configure/etc/conf/plugins.properties b/configure/etc/conf/plugins.properties index 6a51497434..73bf0c04ad 100644 --- a/configure/etc/conf/plugins.properties +++ b/configure/etc/conf/plugins.properties @@ -54,7 +54,7 @@ metaforge-convert-json=convert/metaforge-convert-json/pom.xml metaforge-convert-xml=convert/metaforge-convert-xml/pom.xml metaforge-convert-csv=convert/metaforge-convert-csv/pom.xml metaforge-convert-none=convert/metaforge-convert-none/pom.xml -metaforge-executor-local=executor/metaforge-executor-local/pom.xml +metaforge-executor-metaforge=executor/metaforge-executor-metaforge/pom.xml metaforge-executor-seatunnel=executor/metaforge-executor-seatunnel/pom.xml metaforge-scheduler-local=scheduler/metaforge-scheduler-local/pom.xml metaforge-fs-local=fs/metaforge-fs-local/pom.xml diff --git a/configure/metadata.json b/configure/metadata.json index 14ae91de6a..685a071496 100644 --- a/configure/metadata.json +++ b/configure/metadata.json @@ -3,6 +3,7 @@ "label": "common.plugin.list.name", "description": "common.plugin.list.description", "i18nFormat": true, + "version": "2026.0.0", "children": [ { "key": "metaforge-plugin-mysql", @@ -11,14 +12,31 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/mysql.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "8.0", "5.7" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mysql/2026.0.0/metaforge-plugin-mysql-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "schema", + "ddl", + "index", + "foreignKey", + "trigger", + "dataEdit", + "export", + "erDiagram" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mysql/2026.0.0/metaforge-plugin-mysql-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mysql/2026.0.0/metaforge-plugin-mysql-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-mysql-bin.tar.gz" + } }, { "key": "metaforge-plugin-clickhouse", @@ -27,13 +45,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/clickhouse.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-clickhouse/2026.0.0/metaforge-plugin-clickhouse-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-clickhouse/2026.0.0/metaforge-plugin-clickhouse-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-clickhouse/2026.0.0/metaforge-plugin-clickhouse-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-clickhouse-bin.tar.gz" + } }, { "key": "metaforge-plugin-influxdb", @@ -42,13 +71,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/influxdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-influxdb/2026.0.0/metaforge-plugin-influxdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-influxdb/2026.0.0/metaforge-plugin-influxdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-influxdb/2026.0.0/metaforge-plugin-influxdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-influxdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-solr", @@ -57,13 +97,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/solr.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-solr/2026.0.0/metaforge-plugin-solr-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-solr/2026.0.0/metaforge-plugin-solr-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-solr/2026.0.0/metaforge-plugin-solr-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-solr-bin.tar.gz" + } }, { "key": "metaforge-plugin-cratedb", @@ -72,13 +123,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/cratedb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cratedb/2026.0.0/metaforge-plugin-cratedb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cratedb/2026.0.0/metaforge-plugin-cratedb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cratedb/2026.0.0/metaforge-plugin-cratedb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-cratedb-bin.tar.gz" + } }, { "key": "metaforge-plugin-zookeeper", @@ -87,13 +149,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/zookeeper.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-zookeeper/2026.0.0/metaforge-plugin-zookeeper-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-zookeeper/2026.0.0/metaforge-plugin-zookeeper-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-zookeeper/2026.0.0/metaforge-plugin-zookeeper-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-zookeeper-bin.tar.gz" + } }, { "key": "metaforge-plugin-kafka", @@ -102,13 +173,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/kafka.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kafka/2026.0.0/metaforge-plugin-kafka-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kafka/2026.0.0/metaforge-plugin-kafka-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kafka/2026.0.0/metaforge-plugin-kafka-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-kafka-bin.tar.gz" + } }, { "key": "metaforge-plugin-h2", @@ -117,13 +197,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/h2.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2/2026.0.0/metaforge-plugin-h2-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2/2026.0.0/metaforge-plugin-h2-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2/2026.0.0/metaforge-plugin-h2-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-h2-bin.tar.gz" + } }, { "key": "metaforge-plugin-db2", @@ -132,13 +223,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/db2.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-db2/2026.0.0/metaforge-plugin-db2-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-db2/2026.0.0/metaforge-plugin-db2-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-db2/2026.0.0/metaforge-plugin-db2-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-db2-bin.tar.gz" + } }, { "key": "metaforge-plugin-dm", @@ -147,13 +249,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/dm.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dm/2026.0.0/metaforge-plugin-dm-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dm/2026.0.0/metaforge-plugin-dm-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dm/2026.0.0/metaforge-plugin-dm-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-dm-bin.tar.gz" + } }, { "key": "metaforge-plugin-hdfs", @@ -162,13 +275,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/hdfs.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hdfs/2026.0.0/metaforge-plugin-hdfs-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hdfs/2026.0.0/metaforge-plugin-hdfs-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hdfs/2026.0.0/metaforge-plugin-hdfs-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-hdfs-bin.tar.gz" + } }, { "key": "metaforge-plugin-alioss", @@ -177,13 +299,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/alioss.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-alioss/2026.0.0/metaforge-plugin-alioss-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-alioss/2026.0.0/metaforge-plugin-alioss-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-alioss/2026.0.0/metaforge-plugin-alioss-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-alioss-bin.tar.gz" + } }, { "key": "metaforge-plugin-dremio", @@ -192,13 +323,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/dremio.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dremio/2026.0.0/metaforge-plugin-dremio-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dremio/2026.0.0/metaforge-plugin-dremio-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dremio/2026.0.0/metaforge-plugin-dremio-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-dremio-bin.tar.gz" + } }, { "key": "metaforge-plugin-druid", @@ -207,13 +349,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/druid.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-druid/2026.0.0/metaforge-plugin-druid-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-druid/2026.0.0/metaforge-plugin-druid-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-druid/2026.0.0/metaforge-plugin-druid-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-druid-bin.tar.gz" + } }, { "key": "metaforge-plugin-duckdb", @@ -222,13 +375,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/duckdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-duckdb/2026.0.0/metaforge-plugin-duckdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-duckdb/2026.0.0/metaforge-plugin-duckdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-duckdb/2026.0.0/metaforge-plugin-duckdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-duckdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-pinot", @@ -237,13 +401,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/pinot.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-pinot/2026.0.0/metaforge-plugin-pinot-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-pinot/2026.0.0/metaforge-plugin-pinot-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-pinot/2026.0.0/metaforge-plugin-pinot-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-pinot-bin.tar.gz" + } }, { "key": "metaforge-plugin-cassandra", @@ -252,13 +427,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/cassandra.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cassandra/2026.0.0/metaforge-plugin-cassandra-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cassandra/2026.0.0/metaforge-plugin-cassandra-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-cassandra/2026.0.0/metaforge-plugin-cassandra-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-cassandra-bin.tar.gz" + } }, { "key": "metaforge-plugin-matrixone", @@ -267,13 +451,30 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/matrixone.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-matrixone/2026.0.0/metaforge-plugin-matrixone-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "schema", + "ddl", + "index", + "foreignKey", + "trigger", + "dataEdit", + "export", + "erDiagram" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-matrixone/2026.0.0/metaforge-plugin-matrixone-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-matrixone/2026.0.0/metaforge-plugin-matrixone-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-matrixone-bin.tar.gz" + } }, { "key": "metaforge-plugin-scylladb", @@ -282,13 +483,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/scylladb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-scylladb/2026.0.0/metaforge-plugin-scylladb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-scylladb/2026.0.0/metaforge-plugin-scylladb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-scylladb/2026.0.0/metaforge-plugin-scylladb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-scylladb-bin.tar.gz" + } }, { "key": "metaforge-plugin-paradedb", @@ -297,13 +507,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/paradedb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-paradedb/2026.0.0/metaforge-plugin-paradedb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-paradedb/2026.0.0/metaforge-plugin-paradedb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-paradedb/2026.0.0/metaforge-plugin-paradedb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-paradedb-bin.tar.gz" + } }, { "key": "metaforge-plugin-hive-2x", @@ -312,13 +533,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/hive2x.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "2.x" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hive-2x/2026.0.0/metaforge-plugin-hive-2x-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hive-2x/2026.0.0/metaforge-plugin-hive-2x-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hive-2x/2026.0.0/metaforge-plugin-hive-2x-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-hive-2x-bin.tar.gz" + } }, { "key": "metaforge-plugin-kyuubi", @@ -327,13 +559,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/kyuubi.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kyuubi/2026.0.0/metaforge-plugin-kyuubi-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kyuubi/2026.0.0/metaforge-plugin-kyuubi-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kyuubi/2026.0.0/metaforge-plugin-kyuubi-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-kyuubi-bin.tar.gz" + } }, { "key": "metaforge-plugin-ignite", @@ -342,13 +585,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/ignite.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ignite/2026.0.0/metaforge-plugin-ignite-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ignite/2026.0.0/metaforge-plugin-ignite-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ignite/2026.0.0/metaforge-plugin-ignite-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-ignite-bin.tar.gz" + } }, { "key": "metaforge-plugin-impala", @@ -357,13 +611,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/impala.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-impala/2026.0.0/metaforge-plugin-impala-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-impala/2026.0.0/metaforge-plugin-impala-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-impala/2026.0.0/metaforge-plugin-impala-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-impala-bin.tar.gz" + } }, { "key": "metaforge-plugin-kylin", @@ -372,13 +637,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/kylin.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kylin/2026.0.0/metaforge-plugin-kylin-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kylin/2026.0.0/metaforge-plugin-kylin-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-kylin/2026.0.0/metaforge-plugin-kylin-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-kylin-bin.tar.gz" + } }, { "key": "metaforge-plugin-timescale", @@ -387,13 +663,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/timescale.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-timescale/2026.0.0/metaforge-plugin-timescale-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-timescale/2026.0.0/metaforge-plugin-timescale-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-timescale/2026.0.0/metaforge-plugin-timescale-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-timescale-bin.tar.gz" + } }, { "key": "metaforge-plugin-iotdb", @@ -402,13 +689,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/iotdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-iotdb/2026.0.0/metaforge-plugin-iotdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-iotdb/2026.0.0/metaforge-plugin-iotdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-iotdb/2026.0.0/metaforge-plugin-iotdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-iotdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-monetdb", @@ -417,13 +715,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/monetdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-monetdb/2026.0.0/metaforge-plugin-monetdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-monetdb/2026.0.0/metaforge-plugin-monetdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-monetdb/2026.0.0/metaforge-plugin-monetdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-monetdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-ydb", @@ -432,13 +741,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/ydb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ydb/2026.0.0/metaforge-plugin-ydb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ydb/2026.0.0/metaforge-plugin-ydb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ydb/2026.0.0/metaforge-plugin-ydb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-ydb-bin.tar.gz" + } }, { "key": "metaforge-plugin-neo4j", @@ -447,13 +767,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/neo4j.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-neo4j/2026.0.0/metaforge-plugin-neo4j-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-neo4j/2026.0.0/metaforge-plugin-neo4j-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-neo4j/2026.0.0/metaforge-plugin-neo4j-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-neo4j-bin.tar.gz" + } }, { "key": "metaforge-plugin-oceanbase", @@ -462,13 +791,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/oceanbase.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oceanbase/2026.0.0/metaforge-plugin-oceanbase-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oceanbase/2026.0.0/metaforge-plugin-oceanbase-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oceanbase/2026.0.0/metaforge-plugin-oceanbase-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-oceanbase-bin.tar.gz" + } }, { "key": "metaforge-plugin-oracle", @@ -477,13 +817,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/oracle.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oracle/2026.0.0/metaforge-plugin-oracle-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oracle/2026.0.0/metaforge-plugin-oracle-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-oracle/2026.0.0/metaforge-plugin-oracle-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-oracle-bin.tar.gz" + } }, { "key": "metaforge-plugin-phoenix", @@ -492,13 +843,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/phoenix.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-phoenix/2026.0.0/metaforge-plugin-phoenix-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-phoenix/2026.0.0/metaforge-plugin-phoenix-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-phoenix/2026.0.0/metaforge-plugin-phoenix-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-phoenix-bin.tar.gz" + } }, { "key": "metaforge-plugin-postgresql", @@ -507,13 +869,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/postgresql.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-postgresql/2026.0.0/metaforge-plugin-postgresql-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-postgresql/2026.0.0/metaforge-plugin-postgresql-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-postgresql/2026.0.0/metaforge-plugin-postgresql-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-postgresql-bin.tar.gz" + } }, { "key": "metaforge-plugin-presto", @@ -522,13 +895,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/presto.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-presto/2026.0.0/metaforge-plugin-presto-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-presto/2026.0.0/metaforge-plugin-presto-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-presto/2026.0.0/metaforge-plugin-presto-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-presto-bin.tar.gz" + } }, { "key": "metaforge-plugin-snowflake", @@ -537,13 +921,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/snowflake.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-snowflake/2026.0.0/metaforge-plugin-snowflake-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-snowflake/2026.0.0/metaforge-plugin-snowflake-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-snowflake/2026.0.0/metaforge-plugin-snowflake-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-snowflake-bin.tar.gz" + } }, { "key": "metaforge-plugin-sqlserver", @@ -552,13 +947,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/sqlserver.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-sqlserver/2026.0.0/metaforge-plugin-sqlserver-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-sqlserver/2026.0.0/metaforge-plugin-sqlserver-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-sqlserver/2026.0.0/metaforge-plugin-sqlserver-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-sqlserver-bin.tar.gz" + } }, { "key": "metaforge-plugin-tdengine", @@ -567,13 +973,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/tdengine.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-tdengine/2026.0.0/metaforge-plugin-tdengine-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-tdengine/2026.0.0/metaforge-plugin-tdengine-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-tdengine/2026.0.0/metaforge-plugin-tdengine-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-tdengine-bin.tar.gz" + } }, { "key": "metaforge-plugin-trino", @@ -582,13 +999,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/trino.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-trino/2026.0.0/metaforge-plugin-trino-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-trino/2026.0.0/metaforge-plugin-trino-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-trino/2026.0.0/metaforge-plugin-trino-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-trino-bin.tar.gz" + } }, { "key": "metaforge-plugin-doris", @@ -597,13 +1025,30 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/doris.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-doris/2026.0.0/metaforge-plugin-doris-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "schema", + "ddl", + "index", + "foreignKey", + "trigger", + "dataEdit", + "export", + "erDiagram" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-doris/2026.0.0/metaforge-plugin-doris-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-doris/2026.0.0/metaforge-plugin-doris-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-doris-bin.tar.gz" + } }, { "key": "metaforge-plugin-starrocks", @@ -612,13 +1057,30 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/starrocks.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-starrocks/2026.0.0/metaforge-plugin-starrocks-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "schema", + "ddl", + "index", + "foreignKey", + "trigger", + "dataEdit", + "export", + "erDiagram" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-starrocks/2026.0.0/metaforge-plugin-starrocks-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-starrocks/2026.0.0/metaforge-plugin-starrocks-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-starrocks-bin.tar.gz" + } }, { "key": "metaforge-plugin-hologres", @@ -627,13 +1089,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/hologres.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hologres/2026.0.0/metaforge-plugin-hologres-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hologres/2026.0.0/metaforge-plugin-hologres-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-hologres/2026.0.0/metaforge-plugin-hologres-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-hologres-bin.tar.gz" + } }, { "key": "metaforge-plugin-greptimedb", @@ -642,13 +1115,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/greptimedb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-greptimedb/2026.0.0/metaforge-plugin-greptimedb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-greptimedb/2026.0.0/metaforge-plugin-greptimedb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-greptimedb/2026.0.0/metaforge-plugin-greptimedb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-greptimedb-bin.tar.gz" + } }, { "key": "metaforge-plugin-questdb", @@ -657,13 +1139,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/questdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-questdb/2026.0.0/metaforge-plugin-questdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-questdb/2026.0.0/metaforge-plugin-questdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-questdb/2026.0.0/metaforge-plugin-questdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-questdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-h2-tcp", @@ -672,13 +1163,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/h2tcp.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2-tcp/2026.0.0/metaforge-plugin-h2-tcp-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2-tcp/2026.0.0/metaforge-plugin-h2-tcp-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-h2-tcp/2026.0.0/metaforge-plugin-h2-tcp-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-h2-tcp-bin.tar.gz" + } }, { "key": "metaforge-plugin-ceresdb", @@ -687,13 +1189,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/ceresdb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ceresdb/2026.0.0/metaforge-plugin-ceresdb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ceresdb/2026.0.0/metaforge-plugin-ceresdb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-ceresdb/2026.0.0/metaforge-plugin-ceresdb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-ceresdb-bin.tar.gz" + } }, { "key": "metaforge-plugin-redis", @@ -702,13 +1213,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/redis.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-redis/2026.0.0/metaforge-plugin-redis-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-redis/2026.0.0/metaforge-plugin-redis-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-redis/2026.0.0/metaforge-plugin-redis-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-redis-bin.tar.gz" + } }, { "key": "metaforge-plugin-elasticsearch-8x", @@ -717,13 +1237,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/elasticsearch8x.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "8.x" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-elasticsearch-8x/2026.0.0/metaforge-plugin-elasticsearch-8x-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-elasticsearch-8x/2026.0.0/metaforge-plugin-elasticsearch-8x-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-elasticsearch-8x/2026.0.0/metaforge-plugin-elasticsearch-8x-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-elasticsearch-8x-bin.tar.gz" + } }, { "key": "metaforge-plugin-mongo-atlas", @@ -732,13 +1263,20 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/mongoatlas.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-atlas/2026.0.0/metaforge-plugin-mongo-atlas-2026.0.0-bin.tar.gz" + "features": [ + "query" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-atlas/2026.0.0/metaforge-plugin-mongo-atlas-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-atlas/2026.0.0/metaforge-plugin-mongo-atlas-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-mongo-atlas-bin.tar.gz" + } }, { "key": "metaforge-plugin-mongo-community", @@ -747,13 +1285,22 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/mongocommunity.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-community/2026.0.0/metaforge-plugin-mongo-community-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-community/2026.0.0/metaforge-plugin-mongo-community-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-mongo-community/2026.0.0/metaforge-plugin-mongo-community-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-mongo-community-bin.tar.gz" + } }, { "key": "metaforge-plugin-dolphindb", @@ -762,13 +1309,24 @@ "i18nFormat": true, "type": "Connector", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/plugin/dolphindb.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dolphindb/2026.0.0/metaforge-plugin-dolphindb-2026.0.0-bin.tar.gz" + "features": [ + "query", + "database", + "browse", + "index", + "foreignKey" + ], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dolphindb/2026.0.0/metaforge-plugin-dolphindb-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-plugin-dolphindb/2026.0.0/metaforge-plugin-dolphindb-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-plugin-dolphindb-bin.tar.gz" + } }, { "key": "metaforge-convert-csv", @@ -777,13 +1335,18 @@ "i18nFormat": true, "type": "Converter", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/csv.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-csv/2026.0.0/metaforge-convert-csv-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-csv/2026.0.0/metaforge-convert-csv-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-csv/2026.0.0/metaforge-convert-csv-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/convert/metaforge-convert-csv-bin.tar.gz" + } }, { "key": "metaforge-convert-txt", @@ -792,13 +1355,18 @@ "i18nFormat": true, "type": "Converter", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/txt.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-txt/2026.0.0/metaforge-convert-txt-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-txt/2026.0.0/metaforge-convert-txt-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-txt/2026.0.0/metaforge-convert-txt-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/convert/metaforge-convert-txt-bin.tar.gz" + } }, { "key": "metaforge-convert-json", @@ -807,13 +1375,18 @@ "i18nFormat": true, "type": "Converter", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/json.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-json/2026.0.0/metaforge-convert-json-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-json/2026.0.0/metaforge-convert-json-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-json/2026.0.0/metaforge-convert-json-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/convert/metaforge-convert-json-bin.tar.gz" + } }, { "key": "metaforge-convert-xml", @@ -822,13 +1395,18 @@ "i18nFormat": true, "type": "Converter", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/xml.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-xml/2026.0.0/metaforge-convert-xml-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-xml/2026.0.0/metaforge-convert-xml-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-xml/2026.0.0/metaforge-convert-xml-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/convert/metaforge-convert-xml-bin.tar.gz" + } }, { "key": "metaforge-convert-none", @@ -837,28 +1415,38 @@ "i18nFormat": true, "type": "Converter", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/none.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-none/2026.0.0/metaforge-convert-none-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-none/2026.0.0/metaforge-convert-none-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-convert-none/2026.0.0/metaforge-convert-none-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/convert/metaforge-convert-none-bin.tar.gz" + } }, { - "key": "metaforge-executor-local", - "label": "Local Executor", - "description": "A local execution plugin for MetaForge.", + "key": "metaforge-executor-metaforge", + "label": "MetaForge Executor", + "description": "The built-in MetaForge executor for data sync and migration.", "i18nFormat": true, "type": "Executor", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/executor/local.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-local/2026.0.0/metaforge-executor-local-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-metaforge/2026.0.0/metaforge-executor-metaforge-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-metaforge/2026.0.0/metaforge-executor-metaforge-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/executor/metaforge-executor-metaforge-bin.tar.gz" + } }, { "key": "metaforge-executor-seatunnel", @@ -867,13 +1455,18 @@ "i18nFormat": true, "type": "Executor", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/executor/seatunnel.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-seatunnel/2026.0.0/metaforge-executor-seatunnel-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-seatunnel/2026.0.0/metaforge-executor-seatunnel-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-executor-seatunnel/2026.0.0/metaforge-executor-seatunnel-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/executor/metaforge-executor-seatunnel-bin.tar.gz" + } }, { "key": "metaforge-scheduler-local", @@ -882,13 +1475,18 @@ "i18nFormat": true, "type": "Scheduler", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/scheduler/local.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-scheduler-local/2026.0.0/metaforge-scheduler-local-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-scheduler-local/2026.0.0/metaforge-scheduler-local-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-scheduler-local/2026.0.0/metaforge-scheduler-local-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/scheduler/metaforge-scheduler-local-bin.tar.gz" + } }, { "key": "metaforge-fs-local", @@ -897,13 +1495,18 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/local.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-local/2026.0.0/metaforge-fs-local-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-local/2026.0.0/metaforge-fs-local-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-local/2026.0.0/metaforge-fs-local-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-local-bin.tar.gz" + } }, { "key": "metaforge-fs-qiniu", @@ -912,13 +1515,18 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/qiniu.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-qiniu/2026.0.0/metaforge-fs-qiniu-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-qiniu/2026.0.0/metaforge-fs-qiniu-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-qiniu/2026.0.0/metaforge-fs-qiniu-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-qiniu-bin.tar.gz" + } }, { "key": "metaforge-fs-alioss", @@ -927,13 +1535,18 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/alioss.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-alioss/2026.0.0/metaforge-fs-alioss-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-alioss/2026.0.0/metaforge-fs-alioss-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-alioss/2026.0.0/metaforge-fs-alioss-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-alioss-bin.tar.gz" + } }, { "key": "metaforge-fs-tencent-cos", @@ -942,13 +1555,18 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/tencent-cos.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-tencent-cos/2026.0.0/metaforge-fs-tencent-cos-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-tencent-cos/2026.0.0/metaforge-fs-tencent-cos-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-tencent-cos/2026.0.0/metaforge-fs-tencent-cos-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-tencent-cos-bin.tar.gz" + } }, { "key": "metaforge-fs-amazon-s3", @@ -957,13 +1575,18 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/amazon-s3.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-amazon-s3/2026.0.0/metaforge-fs-amazon-s3-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-amazon-s3/2026.0.0/metaforge-fs-amazon-s3-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-amazon-s3/2026.0.0/metaforge-fs-amazon-s3-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-amazon-s3-bin.tar.gz" + } }, { "key": "metaforge-fs-minio", @@ -972,13 +1595,34 @@ "i18nFormat": true, "type": "FileSystem", "version": "2026.0.0", + "requires": "2026.0.0", "author": "metaforge-community", "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/fs/minio.svg", - "released": "2026-01-28 16:48:33", "supportVersion": [ "ALL" ], - "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-minio/2026.0.0/metaforge-fs-minio-2026.0.0-bin.tar.gz" + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-minio/2026.0.0/metaforge-fs-minio-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-fs-minio/2026.0.0/metaforge-fs-minio-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/fs/metaforge-fs-minio-bin.tar.gz" + } + }, + { + "key": "metaforge-notify-dingtalk", + "label": "DingTalk", + "description": "Sends MetaForge notifications to DingTalk group robots.", + "i18nFormat": false, + "type": "Notify", + "version": "2026.0.0", + "requires": "2026.0.0", + "author": "metaforge-community", + "features": [], + "url": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-notify-dingtalk/2026.0.0/metaforge-notify-dingtalk-2026.0.0-bin.tar.gz", + "sources": { + "maven": "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge-notify-dingtalk/2026.0.0/metaforge-notify-dingtalk-2026.0.0-bin.tar.gz", + "cdn": "https://cdn.north.devlive.org/applications/metaforge/plugins/2026.0.0/plugin/metaforge-notify-dingtalk-bin.tar.gz" + } } ] -} \ No newline at end of file +} diff --git a/configure/publish/build-metadata.mjs b/configure/publish/build-metadata.mjs new file mode 100644 index 0000000000..9e66357a1a --- /dev/null +++ b/configure/publish/build-metadata.mjs @@ -0,0 +1,176 @@ +#!/usr/bin/env node +// Builds the plugin store catalogue (metadata.json) from every plugin module's +// plugin.json, and optionally the ForgeStats release catalog. plugin.json sits +// next to capabilities.json in src/main/resources/org/devlive/metaforge/plugin, +// so it also ships inside the plugin jar and the server can show it offline. +// +// node configure/publish/build-metadata.mjs # refresh configure/metadata.json +// node configure/publish/build-metadata.mjs --dist dist/plugins \ +// --out dist/metadata.json --catalog dist/release-catalog.json \ +// --commit "$GITHUB_SHA" --generation "$GITHUB_RUN_NUMBER" --released now --strict +// +// plugin.json holds what only people know: label, +// description, logo, type, author, supported data source versions (supportVersion), +// the minimum MetaForge version (requires, defaults to the build version), extra +// features and the store order. The build adds the rest: version, released time, +// package size and SHA-256 from the built archive, download sources, and the +// feature list derived from the capabilities the plugin declares in code. +import { createHash } from 'node:crypto' +import { createReadStream, existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const MODULE_ROOTS = ['plugin', 'convert', 'executor', 'fs', 'scheduler', 'notify', 'parser'] +const DECLARATIONS = 'src/main/resources/org/devlive/metaforge/plugin' +const CAPABILITIES = `${DECLARATIONS}/capabilities.json` +const DESCRIPTOR = `${DECLARATIONS}/plugin.json` +const MAVEN_BASE = 'https://repo1.maven.org/maven2' +const CDN_BASE = 'https://cdn.north.devlive.org/applications/metaforge/plugins' +const GROUP_PATH = 'org/devlive/metaforge' + +// Feature codes the store translates (store.feature.). Each is offered +// when the plugin declares any of the listed management capabilities. +const FEATURES = [ + ['query', ['query:execute']], + ['database', ['database:list', 'database:info', 'database:alter']], + ['browse', ['table:list', 'table:info', 'table:read', 'tab:data']], + ['schema', ['table:create', 'table:alter', 'table:drop', 'table:truncate', 'column:create', 'column:modify', 'column:drop', 'column:reorder']], + ['ddl', ['table:statement', 'tab:statement']], + ['index', ['index:read', 'index:create', 'index:drop']], + ['foreignKey', ['foreign-key:read', 'foreign-key:create', 'foreign-key:drop']], + ['trigger', ['trigger:read', 'trigger:create', 'trigger:drop']], + ['dataEdit', ['data:insert', 'data:modify', 'data:delete']], + ['export', ['table:export']], + ['erDiagram', ['tab:er-diagram']], + ['keyValue', ['namespace:list', 'key:list', 'key:read']], + ['objectStorage', ['object:list', 'object:read', 'object:download']], +] + +function args(argv) { + const out = { out: join(ROOT, 'configure/metadata.json') } + for (let i = 0; i < argv.length; i++) { + const flag = argv[i] + if (flag === '--strict') { out.strict = true; continue } + const value = argv[++i] + if (!flag.startsWith('--') || value === undefined) throw new Error(`Unknown or incomplete argument: ${flag}`) + out[flag.slice(2)] = value + } + return out +} + +function projectVersion() { + const pom = readFileSync(join(ROOT, 'pom.xml'), 'utf8') + const project = pom.slice(pom.indexOf('metaforge')) + const match = project.match(/([^<]+)<\/version>/) + if (!match) throw new Error('Cannot read the project version from pom.xml') + return match[1].trim() +} + +// The CDN layout of configure/publish/publish-oss-artifact.sh. +function cdnDirectory(key) { + for (const type of ['executor', 'convert', 'fs', 'parser', 'scheduler']) { + if (key.startsWith(`metaforge-${type}-`)) return type + } + return 'plugin' +} + +function features(moduleDir, extra) { + const file = join(moduleDir, CAPABILITIES) + const declared = new Set(existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')).capabilities ?? [] : []) + const codes = FEATURES.filter(([, caps]) => caps.some(cap => declared.has(cap))).map(([code]) => code) + return [...codes, ...(Array.isArray(extra) ? extra.filter(item => typeof item === 'string' && item.trim()) : [])] +} + +function sha256(file) { + return new Promise((done, fail) => { + const hash = createHash('sha256') + createReadStream(file).on('data', chunk => hash.update(chunk)).on('error', fail).on('end', () => done(hash.digest('hex'))) + }) +} + +function modules() { + const found = [] + for (const root of MODULE_ROOTS) { + const base = join(ROOT, root) + if (!existsSync(base)) continue + for (const name of readdirSync(base)) { + const dir = join(base, name) + if (existsSync(join(dir, DESCRIPTOR)) && existsSync(join(dir, 'pom.xml'))) found.push({ key: name, dir }) + } + } + return found +} + +async function main() { + const opts = args(process.argv.slice(2)) + const version = opts.version ?? projectVersion() + const released = opts.released === 'now' ? new Date().toISOString().replace('T', ' ').replace(/\.\d+Z$/, '') : opts.released + const children = [] + const packages = [] + const missing = [] + for (const { key, dir } of modules()) { + const config = JSON.parse(readFileSync(join(dir, DESCRIPTOR), 'utf8')) + if (typeof config.label !== 'string' || !config.label.trim()) throw new Error(`${key}/${DESCRIPTOR} needs a label`) + const mavenPath = `${GROUP_PATH}/${key}/${version}/${key}-${version}-bin.tar.gz` + const cdnPath = `${version}/${cdnDirectory(key)}/${key}-bin.tar.gz` + const entry = { + key, + label: config.label, + description: config.description, + i18nFormat: config.i18nFormat === true, + type: config.type, + version, + requires: config.requires ?? version, + author: config.author, + logo: config.logo, + released, + supportVersion: Array.isArray(config.supportVersion) ? config.supportVersion : undefined, + features: features(dir, config.features), + // url stays the Maven Central package for MetaForge versions before the + // multi-source store; newer servers pick from sources. + url: `${MAVEN_BASE}/${mavenPath}`, + sources: { maven: `${MAVEN_BASE}/${mavenPath}`, cdn: `${CDN_BASE}/${cdnPath}` }, + order: Number.isFinite(config.order) ? config.order : Number.MAX_SAFE_INTEGER, + } + if (opts.dist) { + const archive = join(resolve(opts.dist), `${key}-bin.tar.gz`) + if (existsSync(archive)) { + entry.size = statSync(archive).size + entry.sha256 = await sha256(archive) + packages.push({ id: key, version, sha256: entry.sha256, size: entry.size, status: 'active', sources: { maven: mavenPath, cdn: cdnPath } }) + } + else { + missing.push(key) + } + } + children.push(entry) + } + if (opts.strict && missing.length) throw new Error(`Missing built packages: ${missing.join(', ')}`) + children.sort((a, b) => a.order - b.order || a.key.localeCompare(b.key)) + for (const entry of children) delete entry.order + const metadata = { + key: 'plugin', + label: 'common.plugin.list.name', + description: 'common.plugin.list.description', + i18nFormat: true, + version, + children: JSON.parse(JSON.stringify(children)), + } + writeFileSync(resolve(opts.out), JSON.stringify(metadata, null, 2) + '\n') + console.log(`metadata: ${children.length} plugin(s) -> ${opts.out}${missing.length ? ` (${missing.length} without built package)` : ''}`) + if (opts.catalog) { + if (!/^[0-9a-f]{40}$/.test(opts.commit ?? '') || !/^[1-9]\d*$/.test(opts.generation ?? '')) { + throw new Error('--catalog needs --commit <40 hex> and --generation ') + } + if (!packages.length) throw new Error('--catalog needs --dist with built packages') + const catalog = { schemaVersion: 2, app: 'metaforge', sourceCommit: opts.commit, generation: Number(opts.generation), plugins: packages } + writeFileSync(resolve(opts.catalog), JSON.stringify(catalog, null, 2) + '\n') + console.log(`catalog: ${packages.length} package(s) -> ${opts.catalog}`) + } +} + +main().catch(error => { + console.error(error.message) + process.exit(1) +}) diff --git a/configure/publish/publish-artifact.sh b/configure/publish/publish-artifact.sh index 51bd3c0563..f6d86208bc 100644 --- a/configure/publish/publish-artifact.sh +++ b/configure/publish/publish-artifact.sh @@ -36,8 +36,11 @@ for file in "$HOME/dist"/*bin.tar.gz; do fi done -# Handle metadata file -METADATA_FILE="${HOME}/configure/metadata.json" +# Handle metadata file: built from the packages uploaded below (size, SHA-256, +# sources, features). Release CI publishes the same file automatically; the +# statistics catalog is imported only by CI, from the Maven Central archives. +METADATA_FILE="${HOME}/dist/metadata.json" +node "${HOME}/configure/publish/build-metadata.mjs" --dist "$PLUGIN_DIR" --out "$METADATA_FILE" --released now || exit 1 if [ -f "$METADATA_FILE" ]; then # Upload new metadata file echo "Uploading metadata file" diff --git a/configure/publish/publish-oss-artifact.sh b/configure/publish/publish-oss-artifact.sh index fb14ae314d..3965f9efa0 100644 --- a/configure/publish/publish-oss-artifact.sh +++ b/configure/publish/publish-oss-artifact.sh @@ -50,8 +50,11 @@ for file in "$HOME/dist"/*bin.tar.gz; do fi done -# Handle metadata file -METADATA_FILE="${HOME}/configure/metadata.json" +# Handle metadata file: built from the packages uploaded below (size, SHA-256, +# sources, features). Release CI publishes the same file automatically; the +# statistics catalog is imported only by CI, from the Maven Central archives. +METADATA_FILE="${HOME}/dist/metadata.json" +node "${HOME}/configure/publish/build-metadata.mjs" --dist "$PLUGIN_DIR" --out "$METADATA_FILE" --released now || exit 1 if [ -f "$METADATA_FILE" ]; then # Upload new metadata file echo "Uploading metadata file" diff --git a/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt b/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt index e6a1a19585..00df35c9f7 100644 --- a/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt +++ b/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt @@ -4,7 +4,9 @@ import com.google.common.base.Preconditions.checkState import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.apache.commons.csv.CSVFormat import org.apache.commons.io.FileUtils @@ -75,6 +77,18 @@ class CsvConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("delimiter", mapOf("zh" to "分隔符", "en" to "Delimiter"), ",", + listOf(choice(",", "逗号 ,", "Comma ,"), choice(";", "分号 ;", "Semicolon ;"), choice("\t", "制表符", "Tab"), choice("|", "竖线 |", "Pipe |"))), + ConvertOption("header", mapOf("zh" to "首行字段名", "en" to "Header row"), "true", + listOf(choice("true", "包含", "Included"), choice("false", "不包含", "Left out"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "empty", + listOf(choice("empty", "空单元格", "Empty cell"), choice("NULL", "写为 NULL", "Write NULL"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -83,23 +97,29 @@ class CsvConvertService : ConvertService log.info("${name()} writer start time [ ${DateUtils.now()} ]") response.headers = request.headers val file = FileConvert.formatFile(request, name()) - FileUtils.writeStringToFile(file, response.headers.joinToString(","), Charsets.UTF_8) + val delimiter = request.option("delimiter", ",").replace("\\t", "\t") + val nullText = if (request.excludeNull) "" else request.option("nulls", "empty").let { if (it == "NULL") "NULL" else "" } + val lines = mutableListOf() + if (request.option("header", "true") != "false") + { + lines.add(response.headers.joinToString(delimiter) { csvCell(it.toString(), delimiter) }) + } // The column types follow the header as a second line. if (request.includeStructure) { - FileUtils.writeStringToFile(file, "\n" + request.headers.indices.joinToString(",") { request.types.getOrNull(it)?.toString() ?: "" }, Charsets.UTF_8, true) + lines.add(request.headers.indices.joinToString(delimiter) { csvCell(request.types.getOrNull(it)?.toString() ?: "", delimiter) }) } val columns = mutableListOf() request.columns .forEach { - FileUtils.writeStringToFile(file, "\n", Charsets.UTF_8, true) when (it) { - is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = ",") { value -> if (value == null && request.excludeNull) "" else value.toString() }, Charsets.UTF_8, true) - else -> FileUtils.writeStringToFile(file, it.toString(), Charsets.UTF_8, true) + is List<*> -> lines.add(it.joinToString(separator = delimiter) { value -> if (value == null) nullText else csvCell(value.toString(), delimiter) }) + else -> lines.add(it.toString()) } } + FileUtils.writeStringToFile(file, lines.joinToString("\n"), Charsets.UTF_8) response.columns = columns log.info("${name()} writer end time [ ${DateUtils.now()} ]") response.successful = true @@ -145,3 +165,13 @@ class CsvConvertService : ConvertService return response } } + +/** A cell quoted when it holds the delimiter, a quote or a line break, quotes doubled (RFC 4180). */ +internal fun csvCell(value: String, delimiter: String): String +{ + return if (value.contains(delimiter) || value.contains('"') || value.contains('\n') || value.contains('\r')) + { + "\"" + value.replace("\"", "\"\"") + "\"" + } + else value +} diff --git a/convert/metaforge-convert-csv/src/main/resources/org/devlive/metaforge/plugin/plugin.json b/convert/metaforge-convert-csv/src/main/resources/org/devlive/metaforge/plugin/plugin.json new file mode 100644 index 0000000000..756b24ecf8 --- /dev/null +++ b/convert/metaforge-convert-csv/src/main/resources/org/devlive/metaforge/plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "label": "Csv Convert", + "description": "A CSV file conversion plugin for MetaForge.", + "i18nFormat": true, + "type": "Converter", + "author": "metaforge-community", + "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/csv.svg", + "supportVersion": [ + "ALL" + ], + "order": 51 +} diff --git a/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt b/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt index 6868f709e4..e8b0515236 100644 --- a/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt +++ b/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt @@ -2,6 +2,8 @@ package org.devlive.metaforge.convert.json import com.fasterxml.jackson.core.JsonEncoding import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.util.DefaultIndenter +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode @@ -9,13 +11,16 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert.formatFile +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.slf4j.LoggerFactory.getLogger import java.io.BufferedReader import java.io.IOException import java.io.InputStreamReader import java.time.LocalDateTime +import java.time.ZoneId import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException import java.util.Objects.requireNonNull @@ -131,6 +136,20 @@ class JsonConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("root", mapOf("zh" to "JSON 根结构", "en" to "JSON root"), "array", + listOf(choice("array", "数组 Array", "Array"), choice("lines", "每行一个对象 JSON Lines", "One object per line (JSON Lines)"))), + ConvertOption("indent", mapOf("zh" to "缩进格式", "en" to "Indentation"), "2", + listOf(choice("0", "不缩进", "None"), choice("2", "2 个空格", "2 spaces"), choice("4", "4 个空格", "4 spaces"))), + ConvertOption("dateFormat", mapOf("zh" to "日期输出格式", "en" to "Date format"), "iso", + listOf(choice("iso", "ISO 8601", "ISO 8601"), choice("datetime", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss"), choice("epoch", "时间戳(毫秒)", "Epoch milliseconds"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "keep", + listOf(choice("keep", "保留 null", "Keep null"), choice("omit", "省略字段", "Leave the field out"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -141,9 +160,20 @@ class JsonConvertService : ConvertService val file = formatFile(request, name()) log.info("${name()} writer file absolute path [ ${file.absolutePath} ]") + val lines = request.option("root", "array") == "lines" + val indent = request.option("indent", "2").toIntOrNull() ?: 2 + val dateFormat = request.option("dateFormat", "iso") + val omitNull = request.excludeNull || request.option("nulls", "keep") == "omit" val factory = JsonFactory() + factory.setRootValueSeparator(if (lines) "\n" else " ") factory.createGenerator(file, JsonEncoding.UTF8).use { generator -> generator.codec = ObjectMapper() + // JSON Lines keeps every record on its own line, so it is never indented. + if (indent > 0 && ! lines) + { + val indenter = DefaultIndenter(" ".repeat(indent), DefaultIndenter.SYS_LF) + generator.prettyPrinter = DefaultPrettyPrinter().withObjectIndenter(indenter).withArrayIndenter(indenter) + } if (request.includeStructure) { generator.writeStartObject() @@ -155,9 +185,19 @@ class JsonConvertService : ConvertService generator.writeEndObject() } generator.writeEndArray() - generator.writeFieldName("rows") + if (lines) + { + generator.writeEndObject() + } + else + { + generator.writeFieldName("rows") + } + } + if (! lines) + { + generator.writeStartArray() } - generator.writeStartArray() request.columns.forEach { column -> generator.writeStartObject() for (headerIndex in request.headers.indices) @@ -167,25 +207,18 @@ class JsonConvertService : ConvertService { is List<*> -> { - val value = column[headerIndex] - if (value == null && request.excludeNull) + val value = dateValue(column[headerIndex], dateFormat) + if (value == null && omitNull) { continue } - if (value is LocalDateTime) - { - generator.writeStringField(header, value.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)) - } - else - { - generator.writeObjectField(header, value) - } + generator.writeObjectField(header, value) } is ObjectNode -> { val value = column.get(header) - if ((value == null || value.isNull) && request.excludeNull) + if ((value == null || value.isNull) && omitNull) { continue } @@ -212,10 +245,13 @@ class JsonConvertService : ConvertService } generator.writeEndObject() } - generator.writeEndArray() - if (request.includeStructure) + if (! lines) { - generator.writeEndObject() + generator.writeEndArray() + if (request.includeStructure) + { + generator.writeEndObject() + } } } @@ -232,7 +268,25 @@ class JsonConvertService : ConvertService return response } - private fun isDateTime(text: String): Boolean + /** A date or time value in the chosen format: ISO 8601, yyyy-MM-dd HH:mm:ss or epoch milliseconds. */ +private fun dateValue(value: Any?, format: String): Any? +{ + val time = when (value) + { + is LocalDateTime -> value + is java.sql.Timestamp -> value.toLocalDateTime() + is java.util.Date -> LocalDateTime.ofInstant(value.toInstant(), ZoneId.systemDefault()) + else -> return value + } + return when (format) + { + "epoch" -> time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() + "datetime" -> time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + else -> time.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + } +} + +private fun isDateTime(text: String): Boolean { return try { diff --git a/convert/metaforge-convert-json/src/main/resources/org/devlive/metaforge/plugin/plugin.json b/convert/metaforge-convert-json/src/main/resources/org/devlive/metaforge/plugin/plugin.json new file mode 100644 index 0000000000..23f06a63c9 --- /dev/null +++ b/convert/metaforge-convert-json/src/main/resources/org/devlive/metaforge/plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "label": "Json Convert", + "description": "A JSON file conversion plugin for MetaForge.", + "i18nFormat": true, + "type": "Converter", + "author": "metaforge-community", + "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/json.svg", + "supportVersion": [ + "ALL" + ], + "order": 53 +} diff --git a/convert/metaforge-convert-none/src/main/resources/org/devlive/metaforge/plugin/plugin.json b/convert/metaforge-convert-none/src/main/resources/org/devlive/metaforge/plugin/plugin.json new file mode 100644 index 0000000000..5a3f6906d3 --- /dev/null +++ b/convert/metaforge-convert-none/src/main/resources/org/devlive/metaforge/plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "label": "None Convert", + "description": "A multidimensional arrays file conversion plugin for MetaForge.", + "i18nFormat": true, + "type": "Converter", + "author": "metaforge-community", + "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/none.svg", + "supportVersion": [ + "ALL" + ], + "order": 55 +} diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt index 70890a4e90..0d7b2fa512 100644 --- a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt @@ -1,5 +1,6 @@ package org.devlive.metaforge.convert +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest import org.devlive.metaforge.convert.model.ConvertResponse import org.devlive.metaforge.plugin.Service @@ -18,6 +19,18 @@ interface ConvertService : Service return "Integrate ${name()} format convert" } + /** Writer settings the plugin offers; values arrive in [ConvertRequest.options]. */ + fun options(): List + { + return listOf() + } + + /** Extension of the files the writer produces. */ + fun extension(): String + { + return name().lowercase() + } + fun format(request: ConvertRequest): ConvertResponse fun formatStream(request: ConvertRequest): ConvertResponse diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt new file mode 100644 index 0000000000..a682054d54 --- /dev/null +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt @@ -0,0 +1,23 @@ +package org.devlive.metaforge.convert.model + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings + +/** + * A setting a convert plugin declares for its writer, shown by the export page as a field: a + * choice among [choices] or free text when there are none. Labels are keyed by language (zh, en). + */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "Read-only description of a plugin setting") +data class ConvertOption( + val name: String, + val labels: Map, + val defaultValue: String, + val choices: List = listOf() +) + +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "Read-only description of a choice") +data class ConvertChoice( + val value: String, + val labels: Map +) + +fun choice(value: String, zh: String, en: String) = ConvertChoice(value, mapOf("zh" to zh, "en" to en)) diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt index 93749726c9..b3f1fcdb25 100644 --- a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt @@ -1,7 +1,9 @@ package org.devlive.metaforge.convert.model +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import java.io.InputStream +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "A request carries the caller's values") class ConvertRequest { private var _headers: List = listOf() @@ -15,6 +17,15 @@ class ConvertRequest // Currently only valid for txt format var delimiter: String? = null + // Values of the plugin's declared options (see ConvertService.options), by option name + var options: Map = mapOf() + + /** An option's value, or [fallback] when it was not given. */ + fun option(name: String, fallback: String): String + { + return options[name]?.takeIf { it.isNotEmpty() } ?: fallback + } + // The format type of the original data var format: String? = null diff --git a/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt b/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt index 54f7c2c49e..c5cbf7f855 100644 --- a/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt +++ b/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt @@ -3,7 +3,9 @@ package org.devlive.metaforge.convert import com.google.common.base.Preconditions.checkArgument import com.google.common.base.Preconditions.checkState import org.devlive.metaforge.common.utils.DateUtils +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.apache.commons.io.FileUtils import org.apache.commons.lang3.ObjectUtils.isNotEmpty @@ -96,12 +98,28 @@ class TxtConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("delimiter", mapOf("zh" to "分隔符", "en" to "Delimiter"), "\t", + listOf(choice("\t", "制表符", "Tab"), choice(",", "逗号 ,", "Comma ,"), choice(" ", "空格", "Space"), choice("|", "竖线 |", "Pipe |"))), + ConvertOption("header", mapOf("zh" to "首行字段名", "en" to "Header row"), "true", + listOf(choice("true", "包含", "Included"), choice("false", "不包含", "Left out"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "empty", + listOf(choice("empty", "留空", "Empty"), choice("NULL", "写为 NULL", "Write NULL"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() try { + // A declared delimiter option wins over the request's delimiter. + request.options["delimiter"]?.takeIf { it.isNotEmpty() }?.let { request.delimiter = it.replace("\\t", "\t") } checkArgument(isNotEmpty(request.delimiter), "Delimiter must not be empty") + val header = request.option("header", "true") != "false" + val nullText = if (request.excludeNull) "" else request.option("nulls", "empty").let { if (it == "NULL") "NULL" else "" } log.info("${name()} writer origin path [ ${request.path} ]") log.info("${name()} writer start time [ ${DateUtils.now()} ]") @@ -111,10 +129,7 @@ class TxtConvertService : ConvertService request.delimiter ?.let { delimiter -> log.info("${name()} writer file headers start") - request.headers - .let { headers -> - FileUtils.writeStringToFile(file, headers.joinToString(separator = delimiter), Charsets.UTF_8) - } + FileUtils.writeStringToFile(file, if (header) request.headers.joinToString(separator = delimiter) else "", Charsets.UTF_8) // The column types follow the header as a second line. if (request.includeStructure) { @@ -129,7 +144,7 @@ class TxtConvertService : ConvertService FileUtils.writeStringToFile(file, "\n", Charsets.UTF_8, true) when (it) { - is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = delimiter) { value -> if (value == null && request.excludeNull) "" else value.toString() }, + is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = delimiter) { value -> value?.toString() ?: nullText }, Charsets.UTF_8, true) else -> FileUtils.writeStringToFile(file, it.toString(), Charsets.UTF_8, true) } diff --git a/convert/metaforge-convert-txt/src/main/resources/org/devlive/metaforge/plugin/plugin.json b/convert/metaforge-convert-txt/src/main/resources/org/devlive/metaforge/plugin/plugin.json new file mode 100644 index 0000000000..89c7b95926 --- /dev/null +++ b/convert/metaforge-convert-txt/src/main/resources/org/devlive/metaforge/plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "label": "Txt Convert", + "description": "A TXT file conversion plugin for MetaForge.", + "i18nFormat": true, + "type": "Converter", + "author": "metaforge-community", + "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/txt.svg", + "supportVersion": [ + "ALL" + ], + "order": 52 +} diff --git a/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt b/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt index 10e180da54..b37c9faa99 100644 --- a/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt +++ b/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt @@ -6,7 +6,9 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert.formatFile +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.slf4j.LoggerFactory.getLogger import java.io.ByteArrayOutputStream @@ -66,6 +68,16 @@ class XmlConvertService : ConvertService TODO("Not yet implemented") } + override fun options(): List + { + return listOf( + ConvertOption("rootElement", mapOf("zh" to "根元素名", "en" to "Root element"), root), + ConvertOption("rowElement", mapOf("zh" to "行元素名", "en" to "Row element"), node), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "keep", + listOf(choice("keep", "写为 null", "Write null"), choice("omit", "省略属性", "Leave the attribute out"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -80,7 +92,9 @@ class XmlConvertService : ConvertService .use { generator -> generator.codec = XmlMapper() val staxWriter = generator.staxWriter - staxWriter.writeStartElement(root) + val rowElement = request.option("rowElement", node) + val omitNull = request.excludeNull || request.option("nulls", "keep") == "omit" + staxWriter.writeStartElement(request.option("rootElement", root)) if (request.includeStructure) { staxWriter.writeStartElement("Columns") @@ -94,7 +108,7 @@ class XmlConvertService : ConvertService } request.columns .forEach { column -> - staxWriter.writeStartElement(node) + staxWriter.writeStartElement(rowElement) for (headerIndex in request.headers.indices) { when (column) @@ -102,7 +116,7 @@ class XmlConvertService : ConvertService is List<*> -> { val value = column[headerIndex] - if (value != null || !request.excludeNull) + if (value != null || ! omitNull) { staxWriter.writeAttribute(request.headers[headerIndex].toString(), value.toString()) } diff --git a/convert/metaforge-convert-xml/src/main/resources/org/devlive/metaforge/plugin/plugin.json b/convert/metaforge-convert-xml/src/main/resources/org/devlive/metaforge/plugin/plugin.json new file mode 100644 index 0000000000..e8007c2faf --- /dev/null +++ b/convert/metaforge-convert-xml/src/main/resources/org/devlive/metaforge/plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "label": "Xml Convert", + "description": "A XML file conversion plugin for MetaForge.", + "i18nFormat": true, + "type": "Converter", + "author": "metaforge-community", + "logo": "http://devlive-cdn.oss-cn-beijing.aliyuncs.com/applications/metaforge/resources/logo/convert/xml.svg", + "supportVersion": [ + "ALL" + ], + "order": 54 +} diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/Plugin.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/Plugin.java index dc9c714e22..efe297be27 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/Plugin.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/Plugin.java @@ -231,6 +231,42 @@ public java.util.List testRuntimeConfigures(java.util.Map values) + { + java.util.List problems = new java.util.ArrayList<>(); + for (org.devlive.metaforge.plugin.configure.RuntimeConfigureField field : runtimeConfigures()) { + String value = values.get(field.getName()); + if (value == null || value.trim().isEmpty()) { + continue; + } + if (field.getType() == org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType.NUMBER) { + try { + Double.parseDouble(value.trim()); + } + catch (NumberFormatException e) { + problems.add(field.getName() + " is not a number: " + value); + } + } + else if (field.getType() == org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType.BOOLEAN + && !"true".equalsIgnoreCase(value.trim()) && !"false".equalsIgnoreCase(value.trim())) { + problems.add(field.getName() + " is not true or false: " + value); + } + else if (field.getType() == org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType.SELECT + && field.getOptions() != null && !field.getOptions().contains(value.trim())) { + problems.add(field.getName() + " must be one of " + field.getOptions() + ": " + value); + } + } + return problems; + } + /** * 声明该插件的数据源表单配置字段(字段名 / 类型 / 必填 / 默认值 / 分组 / pipelines 等)。 * 连接器插件默认返回通用 JDBC 字段(原 default.yaml 语义),转换等其余类型返回 null。 @@ -353,7 +389,7 @@ public T getService(Class serviceClass, String name) } // 获取所有服务 -// Get all services + // Get all services public Set getAllServices(Class serviceClass) { validateInjector(); diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginDescriptor.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginDescriptor.java new file mode 100644 index 0000000000..7edda43f8d --- /dev/null +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginDescriptor.java @@ -0,0 +1,130 @@ +package org.devlive.metaforge.plugin; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.security.CodeSource; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * The store details a plugin ships in {@code org/devlive/metaforge/plugin/plugin.json}, next to its + * capabilities.json: label, description, logo, type, author, supported data source versions, the + * minimum MetaForge version and extra features. The release build turns the same file into the + * store catalogue, so an installed plugin can still describe itself when the catalogue is + * unavailable or does not list it (self-built plugins). + */ +@Slf4j +public final class PluginDescriptor +{ + /** The descriptor path inside every plugin jar or classes directory. */ + public static final String FILE = "org/devlive/metaforge/plugin/plugin.json"; + + private static final Set TEXT = Set.of("label", "description", "logo", "type", "author", "requires"); + private static final Set LISTS = Set.of("supportVersion", "features"); + private static final ObjectMapper JSON = new ObjectMapper(); + + // Read once per plugin class: the file ships inside the plugin and never changes while it is loaded. + private static final ClassValue> DECLARED = new ClassValue<>() + { + @Override + protected Map computeValue(Class type) + { + return load(type); + } + }; + + private PluginDescriptor() + { + } + + /** The descriptor of the plugin the class belongs to; empty when the plugin has none. */ + public static Map of(Class plugin) + { + return plugin == null ? Collections.emptyMap() : DECLARED.get(plugin); + } + + private static Map load(Class plugin) + { + URL descriptor = descriptor(plugin); + if (descriptor == null) { + return Collections.emptyMap(); + } + try { + URLConnection connection = descriptor.openConnection(); + // Plugins are reloaded from their jars; a cached jar would stay open and stale. + connection.setUseCaches(false); + try (InputStream stream = connection.getInputStream()) { + return read(stream); + } + } + catch (FileNotFoundException error) { + return Collections.emptyMap(); + } + catch (IOException error) { + // A broken descriptor only costs the offline store details, never the plugin itself. + log.warn("Ignoring unreadable plugin descriptor {}: {}", descriptor, error.getMessage()); + return Collections.emptyMap(); + } + } + + /** Keeps the known fields with the expected types; anything else is ignored. */ + static Map read(InputStream stream) + throws IOException + { + JsonNode root = JSON.readTree(stream); + if (root == null || !root.isObject()) { + return Collections.emptyMap(); + } + Map out = new LinkedHashMap<>(); + root.fields().forEachRemaining(field -> { + JsonNode value = field.getValue(); + if (TEXT.contains(field.getKey()) && value.isTextual() && !value.asText().isBlank()) { + out.put(field.getKey(), value.asText()); + } + else if (LISTS.contains(field.getKey()) && value.isArray()) { + List items = new ArrayList<>(); + value.forEach(item -> { + if (item.isTextual() && !item.asText().isBlank()) { + items.add(item.asText()); + } + }); + out.put(field.getKey(), Collections.unmodifiableList(items)); + } + else if ("i18nFormat".equals(field.getKey()) && value.isBoolean()) { + out.put(field.getKey(), value.asBoolean()); + } + }); + return Collections.unmodifiableMap(out); + } + + /** + * The plugin.json in the jar or classes directory the class comes from. Every plugin uses the + * same path, so a lookup by name could find another plugin's file on a shared class path. + */ + private static URL descriptor(Class type) + { + CodeSource source = type.getProtectionDomain() == null ? null : type.getProtectionDomain().getCodeSource(); + if (source == null || source.getLocation() == null) { + return null; + } + try { + String location = source.getLocation().toString(); + return location.endsWith(".jar") ? new URL("jar:" + location + "!/" + FILE) : new URL(source.getLocation(), FILE); + } + catch (MalformedURLException error) { + return null; + } + } +} diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginManager.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginManager.java index 20e06ea477..7a2d0caeab 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginManager.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginManager.java @@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; @Slf4j @SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", "EI_EXPOSE_REP2"}) @@ -78,6 +79,13 @@ public class PluginManager // Default version private static final String DEFAULT_VERSION = "1.0.0"; + // 已改名插件的旧名称 → 新名称,仅在按旧名称找不到插件时使用 + // Former plugin names to their current names, used only when nothing is registered under the former name + private static final Map RENAMED_PLUGINS = Map.of( + "Local", "MetaForge", + "LocalExecutor", "MetaForge" + ); + // 插件监视器 // Plugin watcher private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { @@ -202,30 +210,32 @@ public boolean installPlugin(Path sourcePath, String targetDirectory) // Handle installation based on plugin type installed = processPluginInstallation(sourcePath, tempDir); - if (installed) { - // 创建最终目标目录 - // Create final target directory - Path targetPath = config.getPluginsDir().resolve(targetDirectory); - - // 如果目标目录已存在,先备份 - // Backup existing target directory if it exists - if (Files.exists(targetPath)) { - backupExistingPlugin(targetPath); - } + if (!installed) { + return false; + } - // 将临时目录移动到最终位置 - // Move temporary directory to final location + Path targetPath = config.getPluginsDir().resolve(targetDirectory); + Path backup = backupExistingPlugin(targetPath); + boolean loaded = false; + try { moveDirectory(tempDir, targetPath); - - // 加载新安装的插件 - // Load newly installed plugin - loadPluginFromDirectory(targetPath); - + List modules = preparePlugins(targetPath); + if (modules.isEmpty()) { + return false; + } + modules.forEach(this::registerPlugin); + loaded = true; log.info("Successfully installed plugin from {} to {}", sourcePath, targetPath); return true; } - - return false; + finally { + if (!loaded) { + deleteDirectory(targetPath); + if (backup != null) { + moveDirectory(backup, targetPath); + } + } + } } catch (Exception e) { log.error("Failed to install plugin from: {} to {}", sourcePath, targetDirectory, e); @@ -263,7 +273,7 @@ private boolean processPluginInstallation(Path sourcePath, Path tempDir) } // Properties 插件 // Properties plugin - if ("properties".equals(extension)) { + else if ("properties".equals(extension)) { installed.set(installPropertiesPlugin(sourcePath, tempDir)); } // POM 插件 @@ -306,21 +316,21 @@ else if (isSpiPlugin(sourcePath)) { // 备份现有插件 // Backup existing plugin - private void backupExistingPlugin(Path pluginPath) + private Path backupExistingPlugin(Path pluginPath) throws IOException { - if (Files.exists(pluginPath)) { - String timestamp = DateUtils.formatYMDHMS(); - Path backupPath = pluginPath.getParent().resolve(pluginPath.getFileName() + ".backup." + timestamp); - - try { - Files.move(pluginPath, backupPath, StandardCopyOption.REPLACE_EXISTING); - log.info("Created backup of existing plugin: {}", backupPath); - } - catch (IOException e) { - log.error("Failed to create backup of existing plugin: {}", pluginPath, e); - throw e; - } + if (!Files.exists(pluginPath)) { + return null; + } + Path backup = Files.createTempDirectory(pluginPath.getParent(), pluginPath.getFileName() + ".backup."); + try { + Files.move(pluginPath, backup, StandardCopyOption.REPLACE_EXISTING); + log.info("Created backup of existing plugin: {}", backup); + return backup; + } + catch (IOException e) { + Files.deleteIfExists(backup); + throw e; } } @@ -490,43 +500,21 @@ private boolean installSpiPlugin(Path sourcePath, Path targetPath) private boolean installTarPlugin(Path sourcePath, Path tempDir) throws IOException { - // 直接使用 TarPluginLoader 处理 - // Directly use TarPluginLoader to handle - TarPluginLoader tarPluginLoader = new TarPluginLoader(); - // 先加载插件到临时目录 - // Load plugins to temporary directory - List plugins = tarPluginLoader.load(sourcePath, tempDir, config.getParentClassLoaderPackages()); - if (!plugins.isEmpty()) { - // 查找解压后的子目录 - // Find extracted subdirectory - try (Stream paths = Files.list(tempDir)) { - Optional subDir = paths - .filter(Files::isDirectory) - .findFirst(); - - if (subDir.isPresent()) { - // 如果存在子目录,将其内容移动到临时目录根目录 - // If subdirectory exists, move its contents to temp directory root - Path source = subDir.get(); - try (Stream files = Files.list(source)) { - files.forEach(file -> { - try { - Path target = tempDir.resolve(file.getFileName()); - Files.move(file, target); - } - catch (IOException e) { - log.error("Failed to move file: {} to {}", file, tempDir, e); - } - }); - } - // 删除空的子目录 - // Delete empty subdirectory - Files.delete(source); + new TarPluginLoader().extractArchive(sourcePath, tempDir); + List entries; + try (Stream paths = Files.list(tempDir)) { + entries = paths.collect(Collectors.toList()); + } + if (entries.size() == 1 && Files.isDirectory(entries.get(0))) { + Path directory = entries.get(0); + try (Stream files = Files.list(directory)) { + for (Path file : files.collect(Collectors.toList())) { + Files.move(file, tempDir.resolve(file.getFileName())); } } - return true; + Files.delete(directory); } - return false; + return !entries.isEmpty(); } // 复制目录 @@ -655,6 +643,8 @@ private List discoverPluginPaths() try (Stream paths = Files.walk(root, config.getScanDepth() == 0 ? 1 : config.getScanDepth())) { candidates = paths.filter(Files::isDirectory) .filter(path -> !path.equals(root)) + .filter(path -> StreamSupport.stream(root.relativize(path).spliterator(), false) + .noneMatch(segment -> isPluginStagingDirectory(segment.toString()))) .sorted() .collect(Collectors.toList()); } @@ -698,11 +688,9 @@ private void expandPluginPath(Path path, Set paths, Set manifests) } } - // 安装时仍同步加载;启动/监视扫描复用同一初始化过程 - // Installation remains synchronous; startup and watcher scans share preparation - private void loadPluginFromDirectory(Path pluginDir) + private static boolean isPluginStagingDirectory(String name) { - preparePlugins(pluginDir).forEach(this::registerPlugin); + return name.startsWith(TEMP_DIR_PREFIX) || name.contains(".backup.") || name.contains(".unload."); } private List preparePlugins(Path pluginDir) @@ -773,8 +761,13 @@ private void registerPlugin(PluginMetadata metadata) synchronized (installLock) { String name = metadata.getName(); PluginMetadata oldPlugin = plugins.remove(name); - if (oldPlugin != null && oldPlugin.getClassLoader() != metadata.getClassLoader()) { - closePluginClassLoader(oldPlugin); + if (oldPlugin != null) { + if (oldPlugin.getState() == PluginState.STOPPED) { + metadata.setState(PluginState.STOPPED); + } + if (oldPlugin.getClassLoader() != metadata.getClassLoader()) { + closePluginClassLoader(oldPlugin); + } } PluginClassLoader loader = (PluginClassLoader) metadata.getClassLoader(); pluginClassLoaders.put(name, loader); @@ -862,7 +855,16 @@ public void destroy() // Get plugin by name public Optional getPlugin(String name) { - return Optional.ofNullable(plugins.get(name)) + PluginMetadata metadata = plugins.get(name); + if (metadata == null && name != null && RENAMED_PLUGINS.containsKey(name)) { + metadata = plugins.get(RENAMED_PLUGINS.get(name)); + } + if (metadata != null && metadata.getState() == PluginState.STOPPED) { + // 禁用后的插件不再对外提供服务,直到重新启用 + // A stopped plugin is out of service until it is enabled again + return Optional.empty(); + } + return Optional.ofNullable(metadata) .map(info -> (Plugin) info.getInstance()); } @@ -887,6 +889,29 @@ public List getPluginInfos() return new ArrayList<>(plugins.values()); } + // 启用或禁用插件(仅切换运行状态标记,不卸载类加载器) + // Enable or disable a plugin (toggle the runtime flag only; class loaders stay loaded) + public boolean setPluginState(String name, boolean enabled) + { + if (name == null || name.trim().isEmpty()) { + log.warn("Plugin name cannot be null or empty"); + return false; + } + synchronized (installLock) { + PluginMetadata metadata = plugins.get(name); + if (metadata == null && RENAMED_PLUGINS.containsKey(name)) { + metadata = plugins.get(RENAMED_PLUGINS.get(name)); + } + if (metadata == null || metadata.getInstance() == null + || metadata.getState() == PluginState.UNLOADED || metadata.getState() == PluginState.FAILED) { + return false; + } + metadata.setState(enabled ? PluginState.STARTED : PluginState.STOPPED); + log.info("Plugin {} state changed to {}", name, metadata.getState()); + return true; + } + } + // 卸载指定名称的插件 // Unload plugin by name public boolean uninstallPlugin(String name) diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginMetadata.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginMetadata.java index 3b7ca56720..ba4c38673f 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginMetadata.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/PluginMetadata.java @@ -6,6 +6,7 @@ import lombok.Data; import java.nio.file.Path; +import java.util.Map; @Data @Builder @@ -30,4 +31,13 @@ public class PluginMetadata @JsonIgnore private Path location; + + /** + * Store details the plugin ships in its own plugin.json (see {@link PluginDescriptor}), so the + * store can describe an installed plugin without the remote catalogue. + */ + public Map getStore() + { + return instance == null ? Map.of() : PluginDescriptor.of(instance.getClass()); + } } diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureField.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureField.java index a688331f4f..894259f2d4 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureField.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureField.java @@ -1,16 +1,21 @@ package org.devlive.metaforge.plugin.configure; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * 描述一个插件可配置字段。所有插件类型(Executor / Scheduler / FS / …)共用。 * - name: 字段 key,序列化到 JSON 配置中的属性名 - * - type: 字段类型,决定 UI 渲染(文本框 / 数字 / 开关 / 密码) + * - type: 字段类型,决定 UI 渲染(文本框 / 数字 / 开关 / 密码 / 下拉) * - defaultValue: 默认值的字符串表示;调用方按 type 解析 * - description: 字段说明,UI tooltip 用 * - tunable: true = 普通用户在触发任务时可临时覆盖;false = 仅管理员可在系统配置改 + * - options: SELECT 类型的可选值 + * - group: 字段分组 key,UI 按组展示;为空表示不分组 * * Generic configurable-field descriptor used by any plugin category. * - tunable=false fields are administrator-only. @@ -19,6 +24,7 @@ @Data @NoArgsConstructor @AllArgsConstructor +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) public class RuntimeConfigureField { private String name; @@ -26,9 +32,21 @@ public class RuntimeConfigureField private String defaultValue; private String description; private boolean tunable; + private List options; + private String group; public RuntimeConfigureField(String name, RuntimeConfigureFieldType type, String defaultValue, String description) { this(name, type, defaultValue, description, false); } + + public RuntimeConfigureField(String name, RuntimeConfigureFieldType type, String defaultValue, String description, boolean tunable) + { + this(name, type, defaultValue, description, tunable, null, null); + } + + public RuntimeConfigureField(String name, RuntimeConfigureFieldType type, String defaultValue, String description, boolean tunable, String group) + { + this(name, type, defaultValue, description, tunable, null, group); + } } diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureFieldType.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureFieldType.java index 654f076f1a..49e39b46cb 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureFieldType.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/configure/RuntimeConfigureFieldType.java @@ -13,5 +13,6 @@ public enum RuntimeConfigureFieldType STRING, NUMBER, BOOLEAN, - PASSWORD + PASSWORD, + SELECT } diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoader.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoader.java index 599c245387..786e078198 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoader.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoader.java @@ -6,6 +6,7 @@ import org.devlive.metaforge.plugin.utils.VersionUtils; import java.io.File; +import java.io.IOException; import java.lang.reflect.Modifier; import java.net.URL; import java.nio.file.Files; @@ -48,7 +49,6 @@ public List load(Path path, Set parentClassLoaderPackages) // 获取编译后的类路径 // Get compiled classpath Path targetClasses = path.resolve("target/classes"); - Path targetDependencies = path.resolve("target/dependency"); // 如果已编译的类不存在,直接返回空列表 // If compiled classes don't exist, return empty list @@ -59,7 +59,7 @@ public List load(Path path, Set parentClassLoaderPackages) // 创建类加载器 // Create class loader - PluginClassLoader classLoader = createProjectClassLoader(targetClasses, targetDependencies, path, parentClassLoaderPackages); + PluginClassLoader classLoader = createProjectClassLoader(targetClasses, path, parentClassLoaderPackages); // 查找并加载插件类 // Find and load plugin classes @@ -73,35 +73,15 @@ public List load(Path path, Set parentClassLoaderPackages) // 创建项目类加载器 // Create project class loader - private PluginClassLoader createProjectClassLoader(Path targetClasses, Path targetDependencies, Path module, Set parentClassLoaderPackages) + private PluginClassLoader createProjectClassLoader(Path targetClasses, Path module, Set parentClassLoaderPackages) throws Exception { List urls = new ArrayList<>(); - - // 添加编译后的类路径 - // Add compiled classes path log.debug("Adding classes directory to classpath: {}", targetClasses); urls.add(targetClasses.toUri().toURL()); - - // 添加所有依赖jar(如果存在) - // Add all dependency jars (if exist) - if (Files.exists(targetDependencies)) { - log.debug("Adding dependencies from: {}", targetDependencies); - try (Stream paths = Files.walk(targetDependencies)) { - paths.filter(path -> path.toString().endsWith(".jar")) - .forEach(path -> { - try { - log.debug("Adding dependency to classpath: {}", path); - urls.add(path.toUri().toURL()); - } - catch (Exception e) { - log.error("Failed to add dependency jar to classpath: {}", path, e); - } - }); - } - } - else { - log.debug("Dependencies directory not found: {}", targetDependencies); + for (Path jar : dependencyJars(module)) { + log.debug("Adding dependency to classpath: {}", jar); + urls.add(jar.toUri().toURL()); } // Path.getFileName() may return null for empty paths, fall back to @@ -166,4 +146,25 @@ private String getClassName(Path baseDir, Path classFile) return relativePath.replace(File.separatorChar, '.') .replace(".class", ""); } + + /** + * The module's dependency jars: the plugin builds copy them to target/dependencies, while + * `mvn dependency:copy-dependencies` run by hand writes target/dependency. Without them a plugin + * finds only the drivers that happen to be on the server's own class path. + */ + static List dependencyJars(Path module) + throws IOException + { + List jars = new ArrayList<>(); + for (String directory : new String[] {"target/dependencies", "target/dependency"}) { + Path dependencies = module.resolve(directory); + if (!Files.isDirectory(dependencies)) { + continue; + } + try (Stream paths = Files.walk(dependencies)) { + paths.filter(path -> path.toString().endsWith(".jar")).sorted().forEach(jars::add); + } + } + return jars; + } } diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginClassLoader.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginClassLoader.java index 7e0bad871d..e80bdf2e67 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginClassLoader.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginClassLoader.java @@ -67,7 +67,14 @@ protected Class loadClass(String name, boolean resolve) // Add Plugin related packages to parent-first list || (parentFirst && name.startsWith("org.devlive.metaforge.plugin")) ) { - return super.loadClass(name, resolve); + try { + return super.loadClass(name, resolve); + } + catch (ClassNotFoundException e) { + // 父加载器没有时再尝试插件自身的 jar(如驱动类) + // Fall back to the plugin's own jars (e.g. driver classes) + return findClass(name); + } } // 其他所有类先尝试从当前类加载器加载 diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginLoaderFactory.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginLoaderFactory.java index 2c26a0108d..44adae9462 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginLoaderFactory.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/PluginLoaderFactory.java @@ -111,7 +111,17 @@ public static List loadPlugins(Path pluginDir, Set parentClassLo List modules = loader.load(pluginDir, parentClassLoaderPackages); if (modules != null && !modules.isEmpty()) { log.info("Successfully loaded [ {} ] plugin(s) using loader type [ {} ]", modules.size(), type); - modules.forEach(v -> v.setClassLoader(loader.getClass().getName())); + // 开发模式的 pom/编译目录加载器不设置 key,商店只能靠类名匹配目录清单而出现重复行; + // 统一以模块目录名(与 metadata.json 的 key 一致)兜底 + // Development loaders leave the key unset, so the store matched by class name and listed + // plugins twice. Fall back to the module directory name, which equals the catalogue key. + Path module = pluginDir.toString().endsWith("pom.xml") && pluginDir.getParent() != null ? pluginDir.getParent() : pluginDir; + modules.forEach(v -> { + v.setClassLoader(loader.getClass().getName()); + if (v.getKey() == null || v.getKey().isBlank()) { + v.setKey(loader.extractPluginName(module)); + } + }); return modules; } } diff --git a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/TarPluginLoader.java b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/TarPluginLoader.java index 6a54f3954a..2e5addbf14 100644 --- a/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/TarPluginLoader.java +++ b/core/metaforge-plugin/src/main/java/org/devlive/metaforge/plugin/loader/TarPluginLoader.java @@ -8,19 +8,14 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContextBuilder; import org.devlive.metaforge.plugin.Plugin; import org.devlive.metaforge.plugin.PluginContextManager; import org.devlive.metaforge.plugin.SpiType; import org.devlive.metaforge.plugin.utils.PluginClassLoaderUtils; import org.devlive.metaforge.plugin.utils.VersionUtils; -import javax.net.ssl.SSLContext; - import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; @@ -28,9 +23,6 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; @@ -92,43 +84,48 @@ public List load(Path path, Set parentClassLoaderPackages) */ public List load(Path path, Path targetDir, Set parentClassLoaderPackages) { + Path extractDir = targetDir; try { - if (isValidTarPath(path)) { - // 如果是 URL 路径,先下载到本地 - // If it's a URL path, download it first - if (path.toString().startsWith("http") || path.toString().startsWith("https")) { - path = downloadTarFile(path.toString().replace(":/", "://")); - } - - if (isValidTarFile(path)) { - // 使用指定的目录或创建临时目录 - // Use specified directory or create temporary directory - Path extractDir = targetDir != null ? targetDir : createTempDirectory(); - - // 解压 tar 文件 - // Extract tar file - extractTarFile(path, extractDir); - - // 从解压目录加载插件 - // Load plugins from extracted directory - List plugins = loadPluginsFromDirectory(extractDir, parentClassLoaderPackages); - - // 如果使用的是临时目录,则清理 - // Clean up if using temporary directory - if (targetDir == null) { - cleanupTempDirectory(extractDir); - } - - return plugins; - } + if (!isValidTarPath(path)) { + return List.of(); } - - return List.of(); + if (extractDir == null) { + extractDir = createTempDirectory(); + } + extractArchive(path, extractDir); + return loadPluginsFromDirectory(extractDir, parentClassLoaderPackages); } catch (Exception e) { log.error("Failed to load plugins from tar file: {}", path, e); return List.of(); } + finally { + if (targetDir == null && extractDir != null) { + cleanupTempDirectory(extractDir); + } + } + } + + /** Extract without creating a class loader. The manager loads from the final install path. */ + public void extractArchive(Path source, Path destination) + throws IOException + { + if (!isValidTarPath(source)) { + throw new IOException("Invalid plugin archive path"); + } + Path archive = source; + boolean remote = isRemote(source); + if (remote) { + archive = downloadTarFile(source.toString().replaceFirst("^(https?):/+(?=[^/])", "$1://")); + } + try { + extractTarFile(archive, destination); + } + finally { + if (remote) { + Files.deleteIfExists(archive); + } + } } /** @@ -217,24 +214,7 @@ private Path downloadTarFile(String url) Path tempFile = Files.createTempFile(TEMP_DIR_PREFIX, ".tar"); try { - // 创建信任所有证书的 SSL 上下文 - // Create SSL context that trusts all certificates - SSLContext sslContext = SSLContextBuilder.create() - .loadTrustMaterial(null, (cert, authType) -> true) - .build(); - - // 创建允许所有主机名的验证器 - // Create verifier that allows all hostnames - SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory( - sslContext, - NoopHostnameVerifier.INSTANCE - ); - - // 创建 HttpClient - // Create HttpClient - try (CloseableHttpClient httpClient = HttpClients.custom() - .setSSLSocketFactory(sslFactory) - .build()) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { // 创建 GET 请求 // Create GET request HttpGet request = new HttpGet(url); @@ -263,7 +243,7 @@ private Path downloadTarFile(String url) return tempFile; } - catch (IOException | NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + catch (IOException e) { try { Files.deleteIfExists(tempFile); } @@ -298,17 +278,19 @@ private void extractTarFile(Path tarFile, Path destDir) { log.info("Extracting tar file to: {}", destDir); - try (InputStream fileIn = Files.newInputStream(tarFile); - BufferedInputStream buffIn = new BufferedInputStream(fileIn); - GZIPInputStream gzipIn = new GZIPInputStream(buffIn); - TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) { + Path root = destDir.toAbsolutePath().normalize(); + try (BufferedInputStream file = new BufferedInputStream(Files.newInputStream(tarFile)); + TarArchiveInputStream tarIn = new TarArchiveInputStream(archiveStream(file))) { TarArchiveEntry entry; while ((entry = tarIn.getNextEntry()) != null) { if (entry.isDirectory()) { continue; } - Path outPath = destDir.resolve(entry.getName()); + Path outPath = root.resolve(entry.getName()).normalize(); + if (!outPath.startsWith(root) || !entry.isFile()) { + throw new IOException("Invalid plugin archive entry"); + } Files.createDirectories(outPath.getParent()); try (OutputStream out = Files.newOutputStream(outPath)) { @@ -346,28 +328,22 @@ private void cleanupTempDirectory(Path directory) private boolean isValidTarPath(Path path) { String pathStr = path.toString(); - return (pathStr.startsWith("http") || pathStr.startsWith("https")) - && (pathStr.endsWith("tar.gz") || pathStr.endsWith("tar") || pathStr.endsWith("tgz")); + return (isRemote(path) || Files.isRegularFile(path)) + && (pathStr.endsWith(".tar.gz") || pathStr.endsWith(".tgz") || pathStr.endsWith(".tar")); } - /** - * 检查是否合法的 Tar 文件,通过解压文件头检查 - * Check if it's a valid Tar file, by checking the file header - * - * @param path file path - * @return true if it's a valid Tar file - */ - private boolean isValidTarFile(Path path) + private static boolean isRemote(Path path) { - try (InputStream fileIn = Files.newInputStream(path); - BufferedInputStream buffIn = new BufferedInputStream(fileIn); - GZIPInputStream gzipIn = new GZIPInputStream(buffIn)) { - log.debug("Validating tar file: {}", path); - log.debug("Tar file header: {}", gzipIn.readNBytes(512)); - return true; - } - catch (IOException e) { - return false; - } + String value = path.toString(); + return value.startsWith("http:/") || value.startsWith("https:/"); + } + + private static InputStream archiveStream(BufferedInputStream file) + throws IOException + { + file.mark(2); + boolean compressed = file.read() == 0x1f && file.read() == 0x8b; + file.reset(); + return compressed ? new GZIPInputStream(file) : file; } } diff --git a/core/metaforge-plugin/src/test/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoaderTest.java b/core/metaforge-plugin/src/test/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoaderTest.java index 5919b7b0b1..cf7b817770 100644 --- a/core/metaforge-plugin/src/test/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoaderTest.java +++ b/core/metaforge-plugin/src/test/java/org/devlive/metaforge/plugin/loader/CompiledPomPluginLoaderTest.java @@ -15,6 +15,8 @@ import java.nio.file.Path; import java.util.List; import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; public class CompiledPomPluginLoaderTest { @@ -50,6 +52,43 @@ public void compiledPluginsKeepTheirActualClassLoader() plugin.getPluginClassLoader().close(); } + @Test + public void developmentModulesUseTheirDirectoryAsCatalogueKey() + throws Exception + { + // metaforge-fs-local registers as "LocalFs"; only the directory name matches the catalogue key. + Path root = module("LocalFsPlugin", "FILESYSTEM"); + Path module = Files.move(root, root.resolveSibling("metaforge-fs-local")); + List loaded = PluginLoaderFactory.loadPlugins(module.resolve("pom.xml"), Set.of("java.", "org.devlive.metaforge.plugin.")); + Assert.assertEquals(1, loaded.size()); + Assert.assertEquals("metaforge-fs-local", loaded.get(0).getKey()); + loaded.get(0).getPluginClassLoader().close(); + } + + @Test + public void everyPluginReadsTheStoreDescriptorShippedInItsOwnClasses() + throws Exception + { + Path first = module("FirstStorePlugin", "EXECUTOR"); + Path second = module("SecondStorePlugin", "SCHEDULER"); + Path descriptor = first.resolve("target/classes").resolve(org.devlive.metaforge.plugin.PluginDescriptor.FILE); + Files.createDirectories(first.resolve("target/classes/org/devlive/metaforge/plugin")); + Files.writeString(descriptor, "{\"label\":\"First\",\"features\":[\"cron\",7],\"order\":1,\"secret\":\"x\"}"); + Path manifest = temporary.getRoot().toPath().resolve("store.properties"); + Files.writeString(manifest, "first=" + first.resolve("pom.xml") + "\nsecond=" + second.resolve("pom.xml") + "\n"); + PluginManager manager = new PluginManager(PluginConfigure.builder().pluginsDir(manifest).autoCleanup(false).build()); + try { + manager.start(); + java.util.Map> stores = new java.util.HashMap<>(); + manager.getPluginInfos().forEach(plugin -> stores.put(plugin.getName(), plugin.getStore())); + Assert.assertEquals(java.util.Map.of("label", "First", "features", List.of("cron")), stores.get("FirstStore")); + Assert.assertEquals(java.util.Map.of(), stores.get("SecondStore")); + } + finally { + manager.stop(); + } + } + @Test public void aManifestRegistersAllCompiledExecutorsAndSchedulers() throws Exception @@ -69,4 +108,28 @@ public void aManifestRegistersAllCompiledExecutorsAndSchedulers() manager.stop(); } } + + @Test + public void dependencyJarsCopiedByThePluginBuildAreOnTheClassPath() + throws Exception + { + // A plugin build copies its driver to target/dependencies; the plugin must load it from there. + Path root = module("DriverProbePlugin", "EXECUTOR"); + Path sources = temporary.newFolder("driver-sources").toPath(); + Path classes = temporary.newFolder("driver-classes").toPath(); + Path source = sources.resolve("ProbeDriver.java"); + Files.writeString(source, "package fixtures.driver; public class ProbeDriver {}"); + Assert.assertEquals(0, ToolProvider.getSystemJavaCompiler().run(null, null, null, "-d", classes.toString(), source.toString())); + Path dependencies = Files.createDirectories(root.resolve("target/dependencies")); + try (JarOutputStream jar = new JarOutputStream(Files.newOutputStream(dependencies.resolve("probe-driver.jar")))) { + jar.putNextEntry(new JarEntry("fixtures/driver/ProbeDriver.class")); + jar.write(Files.readAllBytes(classes.resolve("fixtures/driver/ProbeDriver.class"))); + jar.closeEntry(); + } + + Assert.assertEquals(List.of(dependencies.resolve("probe-driver.jar")), CompiledPomPluginLoader.dependencyJars(root)); + Plugin plugin = new CompiledPomPluginLoader().load(root, Set.of("java.", "org.devlive.metaforge.plugin.")).get(0); + Assert.assertNotNull(plugin.getPluginClassLoader().loadClass("fixtures.driver.ProbeDriver")); + plugin.getPluginClassLoader().close(); + } } diff --git a/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java index aa90b41bbe..e00ce9c7d2 100644 --- a/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java +++ b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java @@ -13,6 +13,10 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; +import org.springframework.security.web.header.writers.DelegatingRequestMatcherHeaderWriter; +import org.springframework.security.web.header.writers.frameoptions.XFrameOptionsHeaderWriter; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; +import org.springframework.security.web.util.matcher.NegatedRequestMatcher; @Configuration @EnableWebSecurity @@ -64,18 +68,28 @@ public PasswordEncoder passwordEncoder() protected void configure(HttpSecurity http) throws Exception { + // /assets/** is the web UI's build output (scripts, styles, images); browsers fetch CSS images without a token. http.cors().and().csrf().disable() .exceptionHandling().authenticationEntryPoint(unauthorizedHandler).and() .sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS).and() .authorizeRequests() - .antMatchers("/", "/**/*.js", "/**/*.css", "/api/auth/**", "/fonts/**", "/static/**", "/h2-console/**", "/api/v1/captcha", "/api/v1/table/dataDownload/**", "/upload/**", "/api/v1/setup/**") + .antMatchers("/", "/**/*.js", "/**/*.css", "/assets/**", "/api/auth/**", "/fonts/**", "/static/**", "/h2-console/**", "/api/v1/captcha", "/api/v1/table/dataDownload/**", "/upload/**", "/api/v1/setup/**", + // Shared reports: token-addressed public API and the pages that render it + "/api/v1/public/**", "/share/**", "/embed/**") + .permitAll() + // Pages of the history-routed web UI; their API calls stay authenticated + .requestMatchers(new WebUiRequestMatcher()) .permitAll() .requestMatchers(new AdminApiRequestMatcher()) .hasAuthority("ADMIN") .anyRequest() .authenticated(); - // Fixed load denied by X-Frame-Options - http.headers().frameOptions().sameOrigin(); + // Same-origin framing everywhere except embedded report pages, whose CSP frame-ancestors + // carries the sites the report owner allowed. + http.headers().frameOptions().disable() + .addHeaderWriter(new DelegatingRequestMatcherHeaderWriter( + new NegatedRequestMatcher(new AntPathRequestMatcher("/embed/**")), + new XFrameOptionsHeaderWriter(XFrameOptionsHeaderWriter.XFrameOptionsMode.SAMEORIGIN))); http.addFilterBefore(authenticationJwtTokenFilter(), UsernamePasswordAuthenticationFilter.class); } } diff --git a/core/metaforge-security/src/main/java/org/devlive/metaforge/security/WebUiRequestMatcher.java b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/WebUiRequestMatcher.java new file mode 100644 index 0000000000..137108b2ad --- /dev/null +++ b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/WebUiRequestMatcher.java @@ -0,0 +1,48 @@ +package org.devlive.metaforge.security; + +import org.springframework.security.web.util.matcher.RequestMatcher; + +import javax.servlet.http.HttpServletRequest; + +/** + * A browser asking for a page of the web UI. The UI uses history routing (/query, /admin/user ...), + * so every such address returns the UI document and the app routes it; the page itself needs no + * sign-in, the APIs it calls do. API paths, static files and anything with a file extension are not + * pages. + */ +public class WebUiRequestMatcher + implements RequestMatcher +{ + private static final String[] NOT_PAGES = {"/api/", "/assets/", "/static/", "/fonts/", "/upload/", "/h2-console", "/share/", "/embed/"}; + + @Override + public boolean matches(HttpServletRequest request) + { + if (!"GET".equals(request.getMethod()) && !"HEAD".equals(request.getMethod())) { + return false; + } + return isPage(path(request)); + } + + public static boolean isPage(String path) + { + if (path == null || path.isEmpty() || !path.startsWith("/") || "/api".equals(path)) { + return false; + } + for (String prefix : NOT_PAGES) { + if (path.startsWith(prefix)) { + return false; + } + } + // Files carry an extension in their last segment (favicon.ico, robots.txt, index.html). + String last = path.substring(path.lastIndexOf('/') + 1); + return !last.contains("."); + } + + static String path(HttpServletRequest request) + { + String uri = request.getRequestURI(); + String context = request.getContextPath(); + return context != null && !context.isEmpty() && uri.startsWith(context) ? uri.substring(context.length()) : uri; + } +} diff --git a/core/metaforge-server/pom.xml b/core/metaforge-server/pom.xml index ea1309082e..46824f4086 100644 --- a/core/metaforge-server/pom.xml +++ b/core/metaforge-server/pom.xml @@ -201,6 +201,20 @@ + + + org.apache.maven.plugins + maven-jar-plugin + + + application.properties + + + com.github.eirslett frontend-maven-plugin diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java index 4f40d30048..4e5710c43e 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java @@ -1,24 +1,29 @@ package org.devlive.metaforge.server.controller; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.ObjectUtils; import org.devlive.metaforge.captcha.CalculateUtils; import org.devlive.metaforge.captcha.entity.ResultEntity; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.service.SystemSettingService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController() +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) @RequestMapping(value = "/api/v1/captcha") public class CaptchaController { private final InitializerConfigure initializer; + private final SystemSettingService settings; - public CaptchaController(InitializerConfigure initializer) + public CaptchaController(InitializerConfigure initializer, SystemSettingService settings) { this.initializer = initializer; + this.settings = settings; } @GetMapping @@ -28,7 +33,7 @@ public CommonResponse getCaptcha(@RequestParam(value = "timestamp") Long return CommonResponse.failure("timestamp is required"); } // If the verification code is disabled, directly skip the logic of obtaining the verification code - if (!initializer.getCaptchaEnable()) { + if (!settings.auth().isCaptcha()) { return CommonResponse.success(Boolean.FALSE); } ResultEntity entity = CalculateUtils.generate(); diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ConfigureController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ConfigureController.java deleted file mode 100644 index 19f5bf49a7..0000000000 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ConfigureController.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.devlive.metaforge.server.controller; - -import org.devlive.metaforge.common.response.CommonResponse; -import org.devlive.metaforge.service.itransient.configuration.Configuration; -import org.devlive.metaforge.service.service.ConfigureService; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController() -@RequestMapping(value = "/api/v1/configure") -public class ConfigureController -{ - private final ConfigureService service; - - public ConfigureController(ConfigureService service) - { - this.service = service; - } - - @GetMapping(value = "/executor") - public CommonResponse executor() - { - return service.getExecutor(); - } -} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DashboardController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DashboardController.java index 969a1b183c..6fa7af7af4 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DashboardController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DashboardController.java @@ -1,11 +1,24 @@ package org.devlive.metaforge.server.controller; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.DashboardShareBody; import org.devlive.metaforge.service.entity.DashboardEntity; import org.devlive.metaforge.service.repository.DashboardRepository; import org.devlive.metaforge.service.service.DashboardService; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; +import java.util.Map; + @RestController() @RequestMapping(value = "/api/v1/dashboard") public class DashboardController @@ -20,4 +33,52 @@ protected DashboardController(DashboardRepository repository, DashboardService s this.repository = repository; this.service = service; } + + /** Opens a published data screen of the current user as their home page. */ + @PutMapping(value = "home/{code}") + public CommonResponse setHomePage(@PathVariable String code) + { + return service.setHomePage(code); + } + + /** Restores the default home page. */ + @DeleteMapping(value = "home") + public CommonResponse clearHomePage() + { + return service.setHomePage(null); + } + + /** Screens other users share with the signed-in user. */ + @GetMapping(value = "shared") + public CommonResponse> shared(@RequestParam(defaultValue = "") String keyword, + @RequestParam(defaultValue = "updated") String sort) + { + return service.shared(keyword, sort); + } + + /** Users and roles to share with. */ + @GetMapping(value = "share/candidates") + public CommonResponse> shareCandidates(@RequestParam(value = "keyword", required = false) String keyword) + { + return service.shareCandidates(keyword); + } + + @GetMapping(value = "{code}/share") + public CommonResponse getShare(@PathVariable String code) + { + return service.getShare(code); + } + + @PutMapping(value = "{code}/share") + public CommonResponse saveShare(@PathVariable String code, @RequestBody DashboardShareBody body) + { + return service.saveShare(code, body); + } + + /** A dynamic chart of the screen, for anyone the screen is shared with. */ + @GetMapping(value = "{code}/reports/{report}/refresh") + public ResponseEntity>>> refreshChart(@PathVariable String code, @PathVariable String report) + { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(service.refreshChart(code, report)); + } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DataSetController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DataSetController.java index ea855647d7..8d2232d476 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DataSetController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DataSetController.java @@ -6,6 +6,8 @@ import org.devlive.metaforge.service.body.FilterBody; import org.devlive.metaforge.service.body.adhoc.Adhoc; import org.devlive.metaforge.service.dataset.DatasetVirtualColumns; +import org.devlive.metaforge.service.dataset.sync.SyncActiveView; +import org.devlive.metaforge.service.dataset.sync.SyncMetricsView; import org.devlive.metaforge.service.entity.DataSetColumnEntity; import org.devlive.metaforge.service.entity.DataSetEntity; import org.devlive.metaforge.service.entity.DatasetHistoryEntity; @@ -50,6 +52,12 @@ protected DataSetController(DataSetRepository repository, DataSetService service this.initializer = initializer; } + @GetMapping(value = "count") + public CommonResponse count() + { + return service.count(); + } + @PostMapping(value = "validateColumns") public CommonResponse validateColumns(@RequestBody List columns) { @@ -102,7 +110,7 @@ public CommonResponse> getColumnsByCode(@PathVariable } @PutMapping(value = "syncData/{code}") - public CommonResponse syncData(@PathVariable String code, + public CommonResponse syncData(@PathVariable String code, @RequestBody(required = false) java.util.Map overrides) { return service.syncData(code, overrides); @@ -146,6 +154,36 @@ public CommonResponse historyStop(@PathVariable Long id) return this.service.stopHistory(id); } + @GetMapping(value = "sync/active") + public CommonResponse> activeSyncs() + { + return this.service.getActiveSyncs(); + } + + @PutMapping(value = "history/pause/{id}") + public CommonResponse historyPause(@PathVariable Long id) + { + return this.service.pauseHistory(id); + } + + @PutMapping(value = "history/resume/{id}") + public CommonResponse historyResume(@PathVariable Long id) + { + return this.service.resumeHistory(id); + } + + @PutMapping(value = "history/retry/{id}") + public CommonResponse historyRetry(@PathVariable Long id) + { + return this.service.retryHistory(id); + } + + @GetMapping(value = "history/metrics/{id}") + public CommonResponse historyMetrics(@PathVariable Long id) + { + return this.service.getHistoryMetrics(id); + } + @GetMapping(value = "getActuators") public CommonResponse> getActuators() { diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java new file mode 100644 index 0000000000..c0956ad70b --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java @@ -0,0 +1,82 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +/** Export of a database's tables: formats, samples, the plan check, the job and its file. */ +@RestController +@RequestMapping(value = "/api/v1/export") +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class DatabaseExportController +{ + private final DatabaseExportService service; + + public DatabaseExportController(DatabaseExportService service) + { + this.service = service; + } + + @GetMapping(value = "formats") + public CommonResponse>> formats() + { + return this.service.formats(); + } + + @PostMapping(value = "{code}/{database}/sample") + public CommonResponse> sample(@PathVariable String code, @PathVariable String database, @RequestParam String table, + @RequestParam(defaultValue = "3") int rows, @RequestBody ExportPlan plan) + { + return this.service.sample(code, database, plan, table, rows); + } + + @PostMapping(value = "{code}/{database}/check") + public CommonResponse> check(@PathVariable String code, @PathVariable String database, @RequestBody ExportPlan plan) + { + return this.service.check(code, database, plan); + } + + @PostMapping(value = "{code}/{database}/execute") + public CommonResponse> execute(@PathVariable String code, @PathVariable String database, @RequestBody ExportPlan plan) + { + return this.service.execute(code, database, plan); + } + + @GetMapping(value = "jobs/{id}") + public CommonResponse> job(@PathVariable String id) + { + return this.service.job(id); + } + + @PostMapping(value = "jobs/{id}/file") + public ResponseEntity file(@PathVariable String id) + { + CommonResponse response = this.service.file(id); + if (!Boolean.TRUE.equals(response.getStatus()) || response.getData() == null) { + return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(response); + } + TableExportService.ExportFile file = response.getData(); + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(file.getName(), StandardCharsets.UTF_8).build().toString()) + .header(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION) + .body(file.getContent()); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/NotificationController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/NotificationController.java index 45e97f92e4..b4f78d6d47 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/NotificationController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/NotificationController.java @@ -1,25 +1,101 @@ package org.devlive.metaforge.server.controller; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.devlive.metaforge.service.entity.NotificationEntity; -import org.devlive.metaforge.service.repository.NotificationRepository; -import org.devlive.metaforge.service.service.NotificationService; +import lombok.Data; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.notify.NotificationInbox; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -@RestController() +import java.util.Arrays; +import java.util.Map; + +/** The signed-in user's notification center: list, counts, read state and removal of their own notifications. */ +@RestController @RequestMapping(value = "/api/v1/notification") @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") public class NotificationController - extends BaseController { - private final NotificationService service; - private final NotificationRepository repository; + private final NotificationInbox inbox; + + public NotificationController(NotificationInbox inbox) + { + this.inbox = inbox; + } + + @Data + public static class ReadRequest + { + private boolean read = true; + } + + private static Long user() + { + return UserDetailsService.getUser().getId(); + } + + private static boolean knownCategory(String category) + { + return category == null || category.isBlank() || NotificationScene.SYSTEM.equals(category) + || Arrays.stream(NotificationScene.Category.values()).anyMatch(value -> value.name().equals(category)); + } + + @PostMapping(value = "list") + public CommonResponse list(@RequestBody NotificationInbox.Query query) + { + if (!knownCategory(query.getCategory())) { + return CommonResponse.failure("Unknown notification category: " + query.getCategory()); + } + return CommonResponse.success(inbox.page(user(), query)); + } + + @GetMapping(value = "summary") + public CommonResponse summary() + { + return CommonResponse.success(inbox.summary(user())); + } + + @GetMapping(value = "{code}") + public CommonResponse info(@PathVariable(value = "code") String code) + { + return inbox.find(user(), code) + .map(CommonResponse::success) + .orElseGet(() -> CommonResponse.failure("Notification not found")); + } + + @PutMapping(value = "{code}/read") + public CommonResponse mark(@PathVariable(value = "code") String code, @RequestBody ReadRequest request) + { + return inbox.mark(user(), code, request.isRead()) ? CommonResponse.success(true) : CommonResponse.failure("Notification not found"); + } + + @PutMapping(value = "read-all") + public CommonResponse> markAllRead(@RequestParam(value = "category", required = false) String category) + { + if (!knownCategory(category)) { + return CommonResponse.failure("Unknown notification category: " + category); + } + return CommonResponse.success(Map.of("updated", inbox.markAllRead(user(), category))); + } + + @DeleteMapping(value = "{code}") + public CommonResponse delete(@PathVariable(value = "code") String code) + { + return inbox.delete(user(), code) ? CommonResponse.success(true) : CommonResponse.failure("Notification not found"); + } - public NotificationController(NotificationService service, NotificationRepository repository) + @DeleteMapping(value = "read") + public CommonResponse> deleteRead() { - super(repository, service); - this.service = service; - this.repository = repository; + return CommonResponse.success(Map.of("deleted", inbox.deleteRead(user()))); } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PluginController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PluginController.java index e2d4db825f..c4970017ae 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PluginController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PluginController.java @@ -1,10 +1,17 @@ package org.devlive.metaforge.server.controller; +import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.plugin.PluginManager; import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.server.service.PluginReferences; +import org.devlive.metaforge.server.service.PluginStore; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -15,19 +22,39 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.io.IOException; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +@Slf4j @RestController @RequestMapping(value = "/api/v1/plugin") @SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) public class PluginController { private final PluginManager pluginManager; + private final PluginReferences references; + private final PluginStore store; + private final NotificationEventPublisher notifications; - public PluginController(PluginManager pluginManager) + public PluginController(PluginManager pluginManager, PluginReferences references, PluginStore store, NotificationEventPublisher notifications) { this.pluginManager = pluginManager; + this.references = references; + this.store = store; + this.notifications = notifications; + } + + /** Tells the administrator how a plugin change went, once it went through. */ + private CommonResponse notified(CommonResponse response, NotificationScene scene, String name, String title, String content) + { + if (Boolean.TRUE.equals(response.getStatus())) { + notifications.publish(scene, title, content, EntityType.PLUGIN, name, name); + } + return response; } @GetMapping @@ -43,14 +70,93 @@ public CommonResponse> getPlugins(@RequestParam(value = "ha @PostMapping(value = "install") public CommonResponse installPlugin(@RequestBody PluginInstallRequest request) { - return CommonResponse.success(pluginManager.installPlugin(Path.of(request.url), request.name)); + if (request.getName() == null || request.getName().isBlank()) { + return CommonResponse.failure("Plugin package and name are required"); + } + // 带版本的商店安装:经统计服务按配置来源下载、校验 SHA-256/大小后安装并回报下载量; + // 仅有 url 的旧请求保持原行为 + // Store installs with a version download through the statistics service from the configured + // source, verify SHA-256 and size, then report the download. Requests with only a url keep + // the original behaviour. + if (request.getVersion() != null && !request.getVersion().isBlank()) { + PluginStore.Package target = PluginStore.Package.builder() + .key(request.getName()) + .version(request.getVersion()) + .url(request.getUrl()) + .sha256(request.getSha256()) + .size(request.getSize()) + .build(); + try { + return notified(store.install(target, (archive, name) -> pluginManager.installPlugin(archive, name)) + ? CommonResponse.success(true) : CommonResponse.failure("Unable to install the plugin package"), + NotificationScene.PLUGIN_INSTALLED, request.getName(), String.format("插件 %s 已安装", request.getName()), + String.format("插件 %s %s 已从插件商店安装。", request.getName(), request.getVersion())); + } + catch (IOException e) { + log.warn("Store install of {} {} failed: {}", request.getName(), request.getVersion(), e.getMessage()); + return CommonResponse.failure(e.getMessage()); + } + } + if (request.getUrl() == null || request.getUrl().isBlank()) { + return CommonResponse.failure("Plugin package and name are required"); + } + return notified(pluginManager.installPlugin(Path.of(request.getUrl()), request.getName()) + ? CommonResponse.success(true) : CommonResponse.failure("Unable to install the plugin package"), + NotificationScene.PLUGIN_INSTALLED, request.getName(), String.format("插件 %s 已安装", request.getName()), + String.format("插件 %s 已安装。", request.getName())); + } + + // 商店目录由服务端拉取远端清单后返回,浏览器不做跨域请求;拉取失败时前端回退到内置目录 + // The store catalogue is fetched server-side so the browser makes no cross-origin request; + // on failure the frontend falls back to its bundled catalogue + @GetMapping(value = "catalogue") + public CommonResponse catalogue() + { + JsonNode catalogue = store.catalogue(); + return catalogue == null ? CommonResponse.failure("The plugin catalogue is unavailable") : CommonResponse.success(catalogue); + } + + // 商店下载量来自统计服务,未知插件不返回(前端显示为未知而不是 0) + // Store download counts from the statistics service; unknown plugins are absent, never zero + @GetMapping(value = "downloads") + public CommonResponse> downloads(@RequestParam(value = "keys", required = false, defaultValue = "") String keys) + { + List list = Arrays.stream(keys.split(",")) + .map(String::trim) + .filter(key -> !key.isEmpty()) + .distinct() + .limit(500) + .collect(Collectors.toList()); + return CommonResponse.success(store.downloads(list)); } @PreAuthorize(value = "hasAuthority('ADMIN')") @DeleteMapping(value = "uninstall/{name}") public CommonResponse uninstallPlugin(@PathVariable(value = "name") String name) { - return CommonResponse.success(pluginManager.uninstallPlugin(name)); + if (references.isInUse(name, pluginManager.getPluginInfos())) { + return CommonResponse.failure("The plugin is referenced by existing resources"); + } + return notified(pluginManager.uninstallPlugin(name) ? CommonResponse.success(true) : CommonResponse.failure("Unable to uninstall the plugin"), + NotificationScene.PLUGIN_UNINSTALLED, name, String.format("插件 %s 已卸载", name), String.format("插件 %s 已卸载。", name)); + } + + @PreAuthorize(value = "hasAuthority('ADMIN')") + @GetMapping(value = "references") + public CommonResponse>> listReferences() + { + return CommonResponse.success(references.list(pluginManager.getPluginInfos())); + } + + // 启用/禁用同样是改变运行中 JVM 的插件生命周期,必须仅限管理员 + @PreAuthorize(value = "hasAuthority('ADMIN')") + @PostMapping(value = "/{name}/state") + public CommonResponse changeState(@PathVariable(value = "name") String name, + @RequestBody PluginStateRequest request) + { + String state = request.isEnabled() ? "启用" : "停用"; + return notified(pluginManager.setPluginState(name, request.isEnabled()) ? CommonResponse.success(true) : CommonResponse.failure("Unable to change plugin state"), + NotificationScene.PLUGIN_STATE_CHANGED, name, String.format("插件 %s 已%s", name, state), String.format("插件 %s 已%s。", name, state)); } @Data @@ -60,5 +166,16 @@ public static class PluginInstallRequest // 插件名称,默认作为插件的安装目录 // Plugin name, default as the installation directory private String name; + // 商店目录中的版本、SHA-256 与大小(字节);有版本时走统计服务下载并校验 + // Version, SHA-256 and size (bytes) from the store catalogue; a version selects the verified store download + private String version; + private String sha256; + private Long size; + } + + @Data + public static class PluginStateRequest + { + private boolean enabled; } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java new file mode 100644 index 0000000000..12fbb3f9de --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java @@ -0,0 +1,100 @@ +package org.devlive.metaforge.server.controller; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.http.CacheControl; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Anonymous access to shared reports by their share token. Only the saved report definition is + * evaluated; visitors cannot send SQL, queries or report codes. + */ +@RestController +@RequestMapping(value = "/api/v1/public/report") +public class PublicReportController +{ + private final ReportShareService service; + + public PublicReportController(ReportShareService service) + { + this.service = service; + } + + @GetMapping("{token}") + public ResponseEntity> view(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(service.view(token)); + } + + @GetMapping("{token}/refresh") + public ResponseEntity>>> refresh(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.maxAge(Duration.ofSeconds(30))).body(service.refresh(token)); + } + + @GetMapping("{token}/table") + public ResponseEntity> table(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.maxAge(Duration.ofSeconds(30))).body(service.table(token)); + } + + @GetMapping("{token}/download") + public ResponseEntity download(@PathVariable("token") String token) + { + return service.download(token) + .map(entry -> ResponseEntity.ok() + .cacheControl(CacheControl.noStore()) + .contentType(new MediaType("text", "csv", StandardCharsets.UTF_8)) + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment() + .filename(entry.getKey() + ".csv", StandardCharsets.UTF_8).build().toString()) + .body(csv(entry.getValue()))) + .orElseGet(() -> ResponseEntity.notFound().build()); + } + + /** UTF-8 CSV with a byte order mark for spreadsheet apps; cells that would start a formula are quoted as text. */ + static byte[] csv(ReportShareService.Table table) + { + StringBuilder out = new StringBuilder("\uFEFF"); + line(out, new ArrayList<>(table.getHeaders())); + table.getRows().forEach(row -> line(out, row)); + return out.toString().getBytes(StandardCharsets.UTF_8); + } + + private static void line(StringBuilder out, List cells) + { + for (int index = 0; index < cells.size(); index++) { + if (index > 0) { + out.append(','); + } + Object value = cells.get(index); + String cell = value == null ? "" : String.valueOf(value); + if (!(value instanceof Number) && !cell.isEmpty() && "=+-@\t\r".indexOf(cell.charAt(0)) >= 0) { + cell = "'" + cell; + } + if (cell.contains(",") || cell.contains("\"") || cell.contains("\n") || cell.contains("\r")) { + cell = "\"" + cell.replace("\"", "\"\"") + "\""; + } + out.append(cell); + } + out.append("\r\n"); + } + + @GetMapping("{token}/data") + public ResponseEntity> data(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.maxAge(Duration.ofSeconds(30))).body(service.data(token)); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicScreenController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicScreenController.java new file mode 100644 index 0000000000..58c1f61c28 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicScreenController.java @@ -0,0 +1,43 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.service.DashboardService; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +/** + * Anonymous access to a data screen through its public link: the published screen limited to the + * widgets the owner chose, and refreshes of its dynamic charts, which run as the owner. + */ +@RestController +@RequestMapping(value = "/api/v1/public/screen") +public class PublicScreenController +{ + private final DashboardService service; + + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Spring-managed services are intentionally shared singletons") + public PublicScreenController(DashboardService service) + { + this.service = service; + } + + @GetMapping("{token}") + public ResponseEntity>> view(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(service.publicView(token)); + } + + @GetMapping("{token}/reports/{report}/refresh") + public ResponseEntity>>> refresh(@PathVariable("token") String token, @PathVariable("report") String report) + { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(service.publicRefresh(token, report)); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java index e4bfacc833..23abeb1620 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java @@ -1,23 +1,49 @@ package org.devlive.metaforge.server.controller; +import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.entity.ReportEntity; import org.devlive.metaforge.service.repository.ReportRepository; import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.List; +import java.util.Map; + @RestController() @RequestMapping(value = "/api/v1/report") public class ReportController extends BaseController { - private final ReportRepository repository; private final ReportService service; + private final ReportShareService shareService; - protected ReportController(ReportRepository repository, ReportService service) + protected ReportController(ReportRepository repository, ReportService service, ReportShareService shareService) { super(repository, service); - this.repository = repository; this.service = service; + this.shareService = shareService; + } + + // Owner only: public link, iframe embedding, allowed sites and link rotation + @PutMapping("{code}/share") + public CommonResponse share(@PathVariable("code") String code, @RequestBody ReportShareService.ShareRequest request) + { + return shareService.share(code, request); + } + + @GetMapping("{code}/chart-refresh") + public ResponseEntity>>> refreshChart(@PathVariable("code") String code) + { + return ResponseEntity.ok() + .cacheControl(CacheControl.noStore()) + .body(service.refreshChart(code)); } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java new file mode 100644 index 0000000000..b77737688a --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java @@ -0,0 +1,73 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.server.web.WebUiDocument; +import org.devlive.metaforge.service.service.DashboardService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +import java.nio.charset.StandardCharsets; + +/** + * Serves the web UI document for shared reports on real paths, so each response can carry its own + * framing policy: /share/report/{token} stays same-origin only, /embed/report/{token} may be framed + * by the sites the owner allowed (CSP frame-ancestors). The web app routes both to the public view. + */ +@Controller +public class SharedReportPageController +{ + private final ReportShareService service; + private final DashboardService screens; + private final WebUiDocument document; + + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Spring-managed services are intentionally shared singletons") + public SharedReportPageController(ReportShareService service, DashboardService screens, WebUiDocument document) + { + this.service = service; + this.screens = screens; + this.document = document; + } + + @GetMapping("/share/screen/{token}") + public ResponseEntity shareScreen(@PathVariable("token") String token) + { + return page(HttpStatus.OK, "frame-ancestors 'self'"); + } + + @GetMapping("/embed/screen/{token}") + public ResponseEntity embedScreen(@PathVariable("token") String token) + { + // Unknown, private or non-embeddable screens render the "not available" view and refuse framing. + return page(HttpStatus.OK, "frame-ancestors " + screens.publicFrameAncestors(token).orElse("'none'")); + } + + @GetMapping("/share/report/{token}") + public ResponseEntity share(@PathVariable("token") String token) + { + return page(HttpStatus.OK, "frame-ancestors 'self'"); + } + + @GetMapping("/embed/report/{token}") + public ResponseEntity embed(@PathVariable("token") String token) + { + // Unknown, private or non-embeddable reports render the "not available" view and refuse framing. + return page(HttpStatus.OK, "frame-ancestors " + service.frameAncestors(token).orElse("'none'")); + } + + private ResponseEntity page(HttpStatus status, String frameAncestors) + { + return document.html() + .map(html -> ResponseEntity.status(status) + .contentType(new MediaType(MediaType.TEXT_HTML, StandardCharsets.UTF_8)) + .cacheControl(CacheControl.noStore()) + .header("Content-Security-Policy", frameAncestors) + .body(html)) + .orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).contentType(MediaType.TEXT_PLAIN).body("The web UI is not packaged with this server")); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java new file mode 100644 index 0000000000..a8369f9c9a --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java @@ -0,0 +1,95 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.FsSettings; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.OAuthSettings; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** System settings, for administrators only (the /api/v1/admin/** guard). */ +@RestController +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +@RequestMapping(value = "/api/v1/admin/settings") +public class SystemSettingController +{ + private final SystemSettingService service; + + public SystemSettingController(SystemSettingService service) + { + this.service = service; + } + + @GetMapping + public CommonResponse> settings() + { + return CommonResponse.success(service.settings()); + } + + @PutMapping + public CommonResponse> save(@RequestBody SettingsBody body) + { + return service.save(body.getAuth(), body.getMail(), body.getOauth(), body.getFs()); + } + + @PostMapping(value = "mail/test") + public CommonResponse testMail(@RequestBody MailTestBody body) + { + return service.testMail(body.getMail(), body.getTo()); + } + + /** Writes and deletes a probe file with the storage as it is in the form. */ + @PostMapping(value = "fs/test") + public CommonResponse testFs(@RequestBody FsSettings fs) + { + return service.testFs(fs); + } + + /** Tries an executor's or the dataset target's runtime values as they are in the form. */ + @PostMapping(value = "runtime/test") + public CommonResponse testRuntime(@RequestBody RuntimeTestBody body) + { + return service.testRuntime(body.getCategory(), body.getName(), body.getValues()); + } + + @Data + @NoArgsConstructor + public static class RuntimeTestBody + { + private String category; + private String name; + private Map values; + } + + @Data + @NoArgsConstructor + public static class SettingsBody + { + private AuthSettings auth; + private MailSettings mail; + /** Third-party sign-in providers by id; left out to keep them as they are. */ + private Map oauth; + /** File storage; left out to keep it as it is. */ + private FsSettings fs; + } + + /** Tests the server as it is in the form, before it is saved. */ + @Data + @NoArgsConstructor + public static class MailTestBody + { + private MailSettings mail; + private String to; + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/WorkflowController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/WorkflowController.java index 9eb07b7697..6d88db6264 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/WorkflowController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/WorkflowController.java @@ -4,13 +4,17 @@ import org.devlive.metaforge.service.entity.WorkflowEntity; import org.devlive.metaforge.service.repository.WorkflowRepository; import org.devlive.metaforge.service.service.WorkflowService; +import org.devlive.metaforge.service.workflow.WorkflowRun; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.util.List; +import java.util.Map; @RestController @RequestMapping(value = "/api/v1/workflow") @@ -31,6 +35,42 @@ public CommonResponse> log(@PathVariable String code) return service.log(code); } + @GetMapping(value = "/catalog") + public CommonResponse> catalog() + { + return service.catalog(); + } + + @GetMapping(value = "/statistics") + public CommonResponse> statistics() + { + return service.statistics(); + } + + @PostMapping(value = "/validate") + public CommonResponse> validate(@RequestBody WorkflowEntity workflow) + { + return service.validate(workflow); + } + + @GetMapping(value = "/runs/{code}") + public CommonResponse> runs(@PathVariable String code) + { + return service.runs(code); + } + + @GetMapping(value = "/run/{code}/{runId}") + public CommonResponse run(@PathVariable String code, @PathVariable String runId) + { + return service.run(code, runId); + } + + @GetMapping(value = "/log/{code}/{runId}") + public CommonResponse> runLog(@PathVariable String code, @PathVariable String runId) + { + return service.log(code, runId); + } + @PutMapping(value = "stop/{code}") public CommonResponse stop(@PathVariable String code) { @@ -42,4 +82,22 @@ public CommonResponse restart(@PathVariable String code) { return service.restart(code); } + + @PostMapping(value = "query-columns") + public CommonResponse>> queryColumns(@RequestBody Map body) + { + return service.queryColumns(body.get("source"), body.get("query")); + } + + @PostMapping(value = "query-preview") + public CommonResponse> queryPreview(@RequestBody Map body) + { + return service.queryPreview(body.get("source"), body.get("query")); + } + + @PutMapping(value = "publish/{code}") + public CommonResponse publish(@PathVariable String code) + { + return service.publish(code); + } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java index d8c77b129b..2e9fcaf4ba 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java @@ -1,88 +1,127 @@ package org.devlive.metaforge.server.controller.user; import com.fasterxml.jackson.annotation.JsonView; -import org.apache.commons.lang3.ObjectUtils; -import org.devlive.metaforge.captcha.entity.ResultEntity; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.common.view.EntityView; import org.devlive.metaforge.service.body.AuthBody; -import org.devlive.metaforge.service.entity.UserEntity; -import org.devlive.metaforge.service.initializer.InitializerConfigure; -import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.service.AuthService; +import org.devlive.metaforge.service.service.OAuthService; import org.devlive.metaforge.service.validation.ValidationGroup; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseCookie; import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.CookieValue; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.time.Duration; +import java.util.Map; @RestController @RequestMapping("/api/auth") public class AuthController { - private final UserService userService; - private final InitializerConfigure initializer; + /** Keeps the state of a third-party sign-in in the browser that started it. */ + private static final String STATE_COOKIE = "metaforge_oauth_state"; - public AuthController(UserService userService, InitializerConfigure initializer) + private final AuthService authService; + private final OAuthService oauthService; + + public AuthController(AuthService authService, OAuthService oauthService) { - this.userService = userService; - this.initializer = initializer; + this.authService = authService; + this.oauthService = oauthService; } - /** - * Validates the captcha. - * - * @param configure the authentication body containing captcha information - * @return the common response object representing the result of the validation - */ - private CommonResponse validateCaptcha(AuthBody configure) + /** What the sign-in and sign-up pages offer, as set in system settings. */ + @GetMapping("/options") + public CommonResponse> options() { - if (initializer.getCaptchaEnable()) { - if (ObjectUtils.isEmpty(configure.getCaptcha())) { - return CommonResponse.failure("Captcha must not be empty"); - } - ResultEntity entity = initializer.getCache().getIfPresent(configure.getTimestamp()); - if (entity == null) { - return CommonResponse.failure("Captcha does not exists"); - } - if (!entity.getResult().equals(configure.getCaptcha())) { - initializer.getCache().invalidate(configure.getTimestamp()); - return CommonResponse.failure("The calculation result is incorrect"); - } - } - return null; + return CommonResponse.success(authService.options()); } @PostMapping("/signin") public CommonResponse authenticateUser(@RequestBody @Validated(ValidationGroup.Crud.Auth.class) AuthBody configure) { - CommonResponse captchaResponse = validateCaptcha(configure); - if (captchaResponse != null) { - return captchaResponse; - } - - UserEntity user = new UserEntity(); - user.setUsername(configure.getUsername()); - user.setPassword(configure.getPassword()); - return this.userService.authenticate(user); + return authService.signin(configure); } @PostMapping("/signup") @JsonView(value = {EntityView.UserView.class}) public CommonResponse registerUser(@RequestBody @Validated(ValidationGroup.Crud.Create.class) AuthBody configure) { - if (!initializer.getRegistrationEnable()) { - return CommonResponse.failure("Registration is disabled"); - } + return authService.signup(configure); + } + + @PostMapping("/code") + public CommonResponse sendCode(@RequestBody EmailCodeBody configure) + { + return authService.sendCode(configure); + } + + @PostMapping("/reset") + public CommonResponse resetPassword(@RequestBody PasswordResetBody configure) + { + return authService.reset(configure); + } - CommonResponse captchaResponse = validateCaptcha(configure); - if (captchaResponse != null) { - return captchaResponse; + /** Sends the browser to the provider's sign-in page, or back to the sign-in page when the provider is off. */ + @GetMapping("/oauth/{provider}/authorize") + public void authorize(@PathVariable String provider, @RequestParam(required = false) String redirect, HttpServletRequest request, HttpServletResponse response) + throws IOException + { + OAuthService.Redirect target = oauthService.authorize(provider, redirect, base()); + if (target.getState() != null) { + response.addHeader(HttpHeaders.SET_COOKIE, stateCookie(request, target.getState(), Duration.ofMinutes(10))); } + response.sendRedirect(target.getLocation()); + } - UserEntity user = new UserEntity(); - user.setUsername(configure.getUsername()); - user.setPassword(configure.getPassword()); - return this.userService.saveOrUpdate(user); + /** Where the provider sends the browser back; it goes on to the sign-in callback page. */ + @GetMapping("/oauth/{provider}/callback") + public void callback(@PathVariable String provider, @RequestParam(required = false) String code, @RequestParam(required = false) String state, + @RequestParam(required = false) String error, @CookieValue(name = STATE_COOKIE, required = false) String browserState, + HttpServletRequest request, HttpServletResponse response) + throws IOException + { + response.addHeader(HttpHeaders.SET_COOKIE, stateCookie(request, "", Duration.ZERO)); + response.sendRedirect(oauthService.callback(provider, code, state, browserState, error, base())); + } + + /** The token of a finished third-party sign-in, for the ticket the callback page was given. */ + @PostMapping("/oauth/ticket") + public CommonResponse ticket(@RequestBody Map body) + { + return oauthService.redeem(body == null ? null : body.get("ticket")); + } + + private static String base() + { + return ServletUriComponentsBuilder.fromCurrentContextPath().build().toUriString(); + } + + private static String stateCookie(HttpServletRequest request, String value, Duration age) + { + // Lax, so it comes along when the provider sends the browser back. + return ResponseCookie.from(STATE_COOKIE, value) + .httpOnly(true) + .secure(request.isSecure()) + .sameSite("Lax") + .path(request.getContextPath() + "/api/auth/oauth") + .maxAge(age) + .build() + .toString(); } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/NotifySettingsController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/NotifySettingsController.java new file mode 100644 index 0000000000..091ba62a2d --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/NotifySettingsController.java @@ -0,0 +1,52 @@ +package org.devlive.metaforge.server.controller.user; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.notify.UserNotifySettings; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** The signed-in user's own notification channels, their settings and subscribed scenes. */ +@RestController +@RequestMapping(value = "/api/v1/user/notify") +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "The settings service is a Spring-managed singleton") +public class NotifySettingsController +{ + private final UserNotifySettings settings; + + public NotifySettingsController(UserNotifySettings settings) + { + this.settings = settings; + } + + @GetMapping(value = "catalog") + public CommonResponse catalog() + { + return CommonResponse.success(settings.catalog(UserDetailsService.getUser())); + } + + @GetMapping + public CommonResponse> settings() + { + return settings.settings(UserDetailsService.getUser()); + } + + @PutMapping + public CommonResponse> save(@RequestBody List configure) + { + return settings.save(UserDetailsService.getUser(), configure); + } + + @PostMapping(value = "test") + public CommonResponse test(@RequestBody UserNotifySettings.ChannelSetting configure) + { + return settings.test(UserDetailsService.getUser(), configure); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/PluginAuditAdminController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/PluginAuditAdminController.java index a1ef546cc6..3c7a42629b 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/PluginAuditAdminController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/PluginAuditAdminController.java @@ -2,10 +2,12 @@ import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.activity.HeatmapActivity; +import org.devlive.metaforge.service.activity.QueryActivity; import org.devlive.metaforge.service.itransient.ContributionRadar; import org.devlive.metaforge.service.service.PluginAuditService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -27,6 +29,12 @@ public CommonResponse count() return this.pluginAuditService.count(); } + @GetMapping(value = "activity") + public CommonResponse activity(@RequestParam(defaultValue = "7") int days) + { + return this.pluginAuditService.getHomeActivity(days); + } + @GetMapping(value = "contribution") public CommonResponse> contribution() { diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/UserController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/UserController.java index 79f149ce3e..94e449a61a 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/UserController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/UserController.java @@ -1,6 +1,7 @@ package org.devlive.metaforge.server.controller.user; import com.google.common.collect.Sets; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.devlive.metaforge.common.response.CommonResponse; @@ -49,6 +50,7 @@ public class UserController private final UserLogService userLogService; private final RoleRepository roleRepository; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Spring-managed repositories and services are intentionally shared singletons") public UserController(UserRepository repository, UserService service, UserLogService userLogService, RoleRepository roleRepository) { super(repository, service); @@ -145,10 +147,4 @@ public CommonResponse uploadAvatar(@ModelAttribute UploadBody conf { return this.service.uploadAvatar(configure); } - - @PutMapping(value = "change-notify") - public CommonResponse changeNotify(@RequestBody UserEntity configure) - { - return this.service.changeNotify(configure); - } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/runner/DatasetHistoryRecoveryRunner.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/runner/DatasetHistoryRecoveryRunner.java index 1404bfaa27..9147635486 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/runner/DatasetHistoryRecoveryRunner.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/runner/DatasetHistoryRecoveryRunner.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.DateFormatUtils; import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.common.RunStateMachine; import org.devlive.metaforge.service.entity.DatasetHistoryEntity; import org.devlive.metaforge.service.initializer.InstallationState; import org.devlive.metaforge.service.repository.DatasetHistoryRepository; @@ -11,13 +12,12 @@ import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; -import java.util.Arrays; import java.util.Date; import java.util.List; /** * 启动时把上次服务异常退出留下的"未完结"同步历史标记为 INTERRUPTED。 - * 仅处理 RUNNING / CREATED / STOPPING 这些非终态。 + * 处理所有非终态(排队、运行、暂停、取消中、重试等,见 RunStateMachine)。 * 在 SchedulerRunner 之前执行(@Order 较小),避免被即将启动的定时任务"运行中"覆盖。 * * On startup, mark sync-history rows left in transient states by a prior crash as INTERRUPTED. @@ -29,11 +29,7 @@ public class DatasetHistoryRecoveryRunner implements CommandLineRunner { - private static final List TRANSIENT_STATES = Arrays.asList( - RunState.RUNNING, - RunState.CREATED, - RunState.STOPPING - ); + private static final List TRANSIENT_STATES = RunStateMachine.activeStates(); private final DatasetHistoryRepository historyRepository; private final InstallationState installationState; @@ -62,6 +58,11 @@ public void run(String... args) Date updateTime = new Date(); for (DatasetHistoryEntity history : stale) { RunState previous = history.getState(); + // 只有暂停或等待重试时的断点是精确的:运行中崩溃时断点之后可能还提交过行 + // Only a paused or waiting run has an exact checkpoint: a crash mid-run may have committed rows past it. + if (previous != RunState.PAUSED && previous != RunState.RETRYING) { + history.setCheckpoint(null); + } history.setState(RunState.INTERRUPTED); history.setMessage(message); history.setUpdateTime(updateTime); diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginReferences.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginReferences.java new file mode 100644 index 0000000000..1d23b68de1 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginReferences.java @@ -0,0 +1,130 @@ +package org.devlive.metaforge.server.service; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +@Component +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "JdbcTemplate is a shared Spring bean") +public class PluginReferences +{ + private final JdbcTemplate jdbc; + + public PluginReferences(JdbcTemplate jdbc) + { + this.jdbc = jdbc; + } + + /** Missing entries mean the plugin type has no tracked business references. */ + public Map> list(List plugins) + { + Map> references = new LinkedHashMap<>(); + Map>> byType = new LinkedHashMap<>(); + for (PluginMetadata plugin : plugins) { + PluginType type = plugin.getType(); + if (type != PluginType.CONNECTOR && type != PluginType.EXECUTOR && type != PluginType.SCHEDULER) { + continue; + } + List items = new ArrayList<>(); + references.put(plugin.getName(), items); + Map> names = byType.computeIfAbsent(type, key -> new LinkedHashMap<>()); + names.put(normalize(plugin.getName()), items); + } + Map> executors = byType.get(PluginType.EXECUTOR); + if (executors != null && executors.containsKey("metaforge")) { + executors.putIfAbsent("local", executors.get("metaforge")); + executors.putIfAbsent("localexecutor", executors.get("metaforge")); + } + collect(byType.get(PluginType.CONNECTOR), "source", + "SELECT source_type, name FROM metaforge_source ORDER BY name"); + collect(executors, "dataset", "SELECT executor, name FROM metaforge_dataset ORDER BY name"); + collect(executors, "workflow", "SELECT executor, name FROM metaforge_workflow ORDER BY name"); + collect(byType.get(PluginType.SCHEDULER), "dataset", "SELECT scheduler, name FROM metaforge_dataset ORDER BY name"); + return references; + } + + public boolean isInUse(String name, List plugins) + { + PluginMetadata plugin = plugins.stream().filter(item -> item.getName().equals(name)).findFirst().orElse(null); + if (plugin == null) { + return false; + } + String normalized = normalize(name); + if (plugin.getType() == PluginType.CONNECTOR) { + return exists("SELECT COUNT(*) FROM metaforge_source WHERE LOWER(source_type) = ?", normalized); + } + if (plugin.getType() == PluginType.EXECUTOR) { + if (exists("SELECT COUNT(*) FROM metaforge_dataset WHERE LOWER(executor) = ?", normalized) + || exists("SELECT COUNT(*) FROM metaforge_workflow WHERE LOWER(executor) = ?", normalized)) { + return true; + } + if ("metaforge".equals(normalized)) { + for (String alias : List.of("local", "localexecutor")) { + boolean registered = plugins.stream().anyMatch(item -> normalize(item.getName()).equals(alias)); + if (!registered && (exists("SELECT COUNT(*) FROM metaforge_dataset WHERE LOWER(executor) = ?", alias) + || exists("SELECT COUNT(*) FROM metaforge_workflow WHERE LOWER(executor) = ?", alias))) { + return true; + } + } + } + } + if (plugin.getType() == PluginType.SCHEDULER) { + return exists("SELECT COUNT(*) FROM metaforge_dataset WHERE LOWER(scheduler) = ?", normalized); + } + return false; + } + + private boolean exists(String sql, String value) + { + Long count = jdbc.queryForObject(sql, Long.class, value); + return count != null && count > 0; + } + + private void collect(Map> names, String kind, String sql) + { + if (names == null) { + return; + } + jdbc.query(sql, result -> { + List items = names.get(normalize(result.getString(1))); + if (items != null) { + items.add(new Reference(kind, result.getString(2))); + } + }); + } + + private static String normalize(String value) + { + return value == null ? "" : value.toLowerCase(Locale.ROOT); + } + + public static final class Reference + { + private final String kind; + private final String name; + + Reference(String kind, String name) + { + this.kind = kind; + this.name = name; + } + + public String getKind() + { + return kind; + } + + public String getName() + { + return name; + } + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginStore.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginStore.java new file mode 100644 index 0000000000..57b13e01cc --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/service/PluginStore.java @@ -0,0 +1,564 @@ +package org.devlive.metaforge.server.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiPredicate; +import java.util.regex.Pattern; + +/** + * Downloads store packages through the ForgeStats statistics service. + * + *

The service answers a download with a 302 to the chosen source (Maven Central, the + * MetaForge CDN) together with a signed receipt and the official SHA-256 and size. The + * package is verified against those values before installation, and a successful + * installation is reported so the store can show download counts. A self-hosted mirror + * uses the receipt and checksums but downloads from the mirror. When the service is + * unreachable or does not list the version, the package is downloaded directly and + * verified against the catalogue values, so installing never depends on statistics.

+ */ +@Slf4j +@Component +public class PluginStore +{ + static final long MAX_PACKAGE_SIZE = 512L * 1024 * 1024; + private static final Pattern KEY = Pattern.compile("^[a-z0-9][a-z0-9._-]{0,190}$"); + private static final Pattern VERSION = Pattern.compile("^[0-9A-Za-z][0-9A-Za-z.+-]{0,63}$"); + private static final Pattern SHA256 = Pattern.compile("^[0-9a-f]{64}$"); + private static final List SOURCES = List.of("maven", "cdn"); + private static final Duration COUNTS_TTL = Duration.ofSeconds(60); + private static final Duration CATALOGUE_TTL = Duration.ofSeconds(60); + private static final int MAX_CATALOGUE_SIZE = 16 * 1024 * 1024; + + private final String statsUrl; + private final String app; + private final String source; + private final String mirror; + private final String metadataUrl; + private final HttpClient http; + // Redirect targets must be HTTPS; tests against a local server relax this. + private final boolean requireHttps; + private final ObjectMapper json = new ObjectMapper(); + private final SecureRandom random = new SecureRandom(); + private final Map counts = new ConcurrentHashMap<>(); + private volatile CachedCatalogue catalogue; + + // Two constructors: Spring must be told which one to use, or it looks for a no-argument one. + @Autowired + public PluginStore(@Value("${plugin.store.stats-url:https://forgestats.devlive.org}") String statsUrl, + @Value("${plugin.store.app:metaforge}") String app, + @Value("${plugin.store.source:maven}") String source, + @Value("${plugin.store.mirror:}") String mirror, + @Value("${plugin.store.metadata-url:https://cdn.north.devlive.org/applications/metaforge/metadata/metadata.json}") String metadataUrl) + { + this(statsUrl, app, source, mirror, metadataUrl, HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)) + .build(), true); + } + + PluginStore(String statsUrl, String app, String source, String mirror, String metadataUrl, HttpClient http, boolean requireHttps) + { + this.requireHttps = requireHttps; + this.statsUrl = trimSlash(statsUrl); + this.app = app == null ? "" : app.trim(); + this.source = source == null || source.isBlank() ? "maven" : source.trim().toLowerCase(Locale.ROOT); + this.mirror = trimSlash(mirror); + this.metadataUrl = metadataUrl == null ? "" : metadataUrl.trim(); + this.http = http; + if ("mirror".equals(this.source) && this.mirror.isEmpty()) { + log.warn("plugin.store.source=mirror needs plugin.store.mirror; downloads use the default source"); + } + } + + /** What the store asks to install: the catalogue entry of one plugin version. */ + @Data + @Builder + public static class Package + { + private String key; + private String version; + private String url; + private String sha256; + private Long size; + } + + @Data + @Builder + static class Receipt + { + private String attemptId; + private String token; + private String sha256; + private long size; + private String location; + } + + /** Thrown when a package cannot be fetched or fails verification. */ + public static class StoreException + extends IOException + { + public StoreException(String message) + { + super(message); + } + } + + /** + * Fetches and verifies a package, hands the local archive to {@code installer} and reports a + * successful installation. The archive is deleted afterwards. + */ + public boolean install(Package target, BiPredicate installer) + throws IOException + { + if (target.getKey() == null || !KEY.matcher(target.getKey()).matches()) { + throw new StoreException("Invalid plugin key"); + } + if (target.getVersion() == null || !VERSION.matcher(target.getVersion()).matches()) { + throw new StoreException("Invalid plugin version"); + } + String expectedSha = target.getSha256() == null ? null : target.getSha256().toLowerCase(Locale.ROOT); + if (expectedSha != null && !SHA256.matcher(expectedSha).matches()) { + throw new StoreException("Invalid package checksum"); + } + Path directory = Files.createTempDirectory("metaforge-store-"); + Path archive = directory.resolve(target.getKey() + "-bin.tar.gz"); + try { + Optional receipt = fetchCounted(target, archive); + if (receipt.isEmpty()) { + if (target.getUrl() == null || target.getUrl().isBlank()) { + throw new StoreException("The statistics service is unavailable and the catalogue has no package URL"); + } + download(URI.create(target.getUrl()), archive, expectedSha, target.getSize()); + } + else if (expectedSha != null && !expectedSha.equals(receipt.get().getSha256())) { + // The catalogue and the release record disagree; trust neither silently. + throw new StoreException("The package checksum differs from the store catalogue"); + } + if (!installer.test(archive, target.getKey())) { + return false; + } + receipt.ifPresent(this::report); + return true; + } + finally { + deleteQuietly(archive); + deleteQuietly(directory); + } + } + + /** Counted download through the statistics service; empty when it cannot be used. */ + private Optional fetchCounted(Package target, Path archive) + throws StoreException + { + if (statsUrl.isEmpty() || app.isEmpty()) { + return Optional.empty(); + } + String attempt = randomHex(16); + boolean useMirror = "mirror".equals(source) && !mirror.isEmpty(); + List order = new ArrayList<>(); + if (useMirror) { + order.add(""); + } + else { + order.add(SOURCES.contains(source) ? source : "maven"); + SOURCES.stream().filter(name -> !order.contains(name)).forEach(order::add); + } + StoreException verification = null; + for (String name : order) { + Optional receipt = issue(target, name, attempt); + if (receipt.isEmpty()) { + // Unknown version, unreachable service: not counted, but still installable. + return Optional.empty(); + } + URI location = useMirror ? URI.create(mirror + "/" + cdnPath(target.getKey(), target.getVersion())) + : URI.create(receipt.get().getLocation()); + try { + download(location, archive, receipt.get().getSha256(), receipt.get().getSize()); + return receipt; + } + catch (StoreException e) { + verification = e; + log.warn("Plugin {} {} from source [ {} ] failed: {}", target.getKey(), target.getVersion(), name.isEmpty() ? "mirror" : name, e.getMessage()); + } + } + throw verification; + } + + Optional issue(Package target, String sourceName, String attempt) + { + String path = String.format("%s/v1/apps/%s/plugins/%s/versions/%s/download%s", statsUrl, app, target.getKey(), target.getVersion(), + sourceName.isEmpty() ? "" : "?source=" + sourceName); + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(path)) + .timeout(Duration.ofSeconds(8)) + .header("Idempotency-Key", attempt) + .header("User-Agent", "metaforge-server") + .GET() + .build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() != 302) { + log.info("Statistics service answered {} for {} {}; downloading without counting", response.statusCode(), target.getKey(), target.getVersion()); + return Optional.empty(); + } + String id = response.headers().firstValue("X-Forge-Attempt-Id").orElse(""); + String token = response.headers().firstValue("X-Forge-Receipt").orElse(""); + String sha = response.headers().firstValue("X-Forge-Sha256").orElse(""); + long size = Long.parseLong(response.headers().firstValue("X-Forge-Size").orElse("0")); + String location = response.headers().firstValue("Location").orElse(""); + if (!attempt.equals(id) || token.isEmpty() || !SHA256.matcher(sha).matches() || size <= 0 || size > MAX_PACKAGE_SIZE + || !(location.startsWith("https://") || (!requireHttps && location.startsWith("http://")))) { + log.warn("Statistics service returned an incomplete download for {} {}", target.getKey(), target.getVersion()); + return Optional.empty(); + } + return Optional.of(Receipt.builder().attemptId(id).token(token).sha256(sha).size(size).location(location).build()); + } + catch (IOException | IllegalArgumentException e) { + log.info("Statistics service unavailable for {} {}: {}", target.getKey(), target.getVersion(), e.getMessage()); + return Optional.empty(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Optional.empty(); + } + } + + /** Downloads to {@code archive}, enforcing the expected size and SHA-256 when known. */ + void download(URI location, Path archive, String sha256, Long size) + throws StoreException + { + try { + HttpRequest request = HttpRequest.newBuilder(location) + .timeout(Duration.ofMinutes(10)) + .header("User-Agent", "metaforge-server") + .GET() + .build(); + HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + if (response.statusCode() != 200) { + throw new StoreException("Package download failed with HTTP " + response.statusCode()); + } + long limit = size != null && size > 0 ? size : MAX_PACKAGE_SIZE; + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + long total = 0; + byte[] buffer = new byte[64 * 1024]; + try (OutputStream out = Files.newOutputStream(archive)) { + for (int read = body.read(buffer); read != -1; read = body.read(buffer)) { + total += read; + if (total > limit) { + throw new StoreException("The package is larger than its published size"); + } + digest.update(buffer, 0, read); + out.write(buffer, 0, read); + } + } + if (size != null && size > 0 && total != size) { + throw new StoreException("The package size differs from its published size"); + } + String actual = hex(digest.digest()); + if (sha256 != null && !sha256.equals(actual)) { + throw new StoreException("The package checksum does not match"); + } + } + } + catch (StoreException e) { + deleteQuietly(archive); + throw e; + } + catch (IOException | IllegalArgumentException | NoSuchAlgorithmException e) { + deleteQuietly(archive); + throw new StoreException("Package download failed: " + e.getMessage()); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + deleteQuietly(archive); + throw new StoreException("Package download interrupted"); + } + } + + /** Reports a verified installation; failures only cost one count. */ + void report(Receipt receipt) + { + String body = String.format("{\"sha256\":\"%s\",\"size\":%d}", receipt.getSha256(), receipt.getSize()); + URI uri = URI.create(String.format("%s/v1/apps/%s/downloads/%s/complete", statsUrl, app, receipt.getAttemptId())); + for (int attempt = 0; attempt < 3; attempt++) { + try { + HttpRequest request = HttpRequest.newBuilder(uri) + .timeout(Duration.ofSeconds(5)) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + receipt.getToken()) + .header("User-Agent", "metaforge-server") + .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)) + .build(); + int status = http.send(request, HttpResponse.BodyHandlers.discarding()).statusCode(); + if (status == 200) { + return; + } + if (status != 429 && status < 500) { + log.warn("Statistics service rejected the download report with HTTP {}", status); + return; + } + } + catch (IOException e) { + log.debug("Download report failed: {}", e.getMessage()); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + Thread.sleep(200L * (attempt + 1)); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + log.warn("Download report for attempt {} was not delivered", receipt.getAttemptId()); + } + + /** + * The remote store catalogue, served to the browser through the controller so the store + * never makes cross-origin requests. Cached briefly; {@code null} when the URL is unset + * or the catalogue cannot be fetched, which lets the store fall back to its bundled copy. + */ + public JsonNode catalogue() + { + if (metadataUrl.isEmpty()) { + return null; + } + CachedCatalogue cached = catalogue; + if (cached != null && System.nanoTime() - cached.at < CATALOGUE_TTL.toNanos()) { + return cached.value; + } + synchronized (this) { + cached = catalogue; + if (cached != null && System.nanoTime() - cached.at < CATALOGUE_TTL.toNanos()) { + return cached.value; + } + JsonNode fetched = fetchCatalogue(); + if (fetched != null) { + catalogue = new CachedCatalogue(fetched, System.nanoTime()); + } + return fetched; + } + } + + private JsonNode fetchCatalogue() + { + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(metadataUrl)) + .timeout(Duration.ofSeconds(8)) + .header("User-Agent", "metaforge-server") + .GET() + .build(); + HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + log.info("The plugin catalogue answered {} for {}", response.statusCode(), metadataUrl); + return null; + } + String body = response.body(); + if (body == null || body.isBlank()) { + log.warn("The plugin catalogue is empty: {}", metadataUrl); + return null; + } + if (body.length() > MAX_CATALOGUE_SIZE) { + log.warn("The plugin catalogue exceeds {} bytes: {}", MAX_CATALOGUE_SIZE, metadataUrl); + return null; + } + JsonNode value = json.readTree(body); + if (!value.isObject()) { + log.warn("The plugin catalogue is not a JSON object: {}", metadataUrl); + return null; + } + return value; + } + catch (IOException | IllegalArgumentException e) { + log.info("The plugin catalogue is unavailable: {}", e.getMessage()); + return null; + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + + /** + * Total confirmed downloads per plugin key, as decimal strings. Keys the service does not + * know, or every key when it is unreachable, are absent: the store shows them as unknown. + */ + public Map downloads(Collection keys) + { + Map out = new LinkedHashMap<>(); + if (statsUrl.isEmpty() || app.isEmpty()) { + return out; + } + long now = System.nanoTime(); + List missing = new ArrayList<>(); + for (String key : keys) { + if (key == null || !KEY.matcher(key).matches()) { + continue; + } + CachedCount cached = counts.get(key); + if (cached != null && now - cached.at < COUNTS_TTL.toNanos()) { + if (cached.value != null) { + out.put(key, cached.value); + } + } + else if (!missing.contains(key)) { + missing.add(key); + } + } + for (int from = 0; from < missing.size(); from += 100) { + List batch = missing.subList(from, Math.min(missing.size(), from + 100)); + Map fetched = fetchCounts(batch); + if (fetched == null) { + break; + } + for (String key : batch) { + String value = fetched.get(key); + counts.put(key, new CachedCount(value, now)); + if (value != null) { + out.put(key, value); + } + } + } + return out; + } + + private Map fetchCounts(List keys) + { + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(String.format("%s/v1/apps/%s/stats?ids=%s", statsUrl, app, String.join(",", keys)))) + .timeout(Duration.ofSeconds(5)) + .header("User-Agent", "metaforge-server") + .GET() + .build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + return null; + } + Map out = new LinkedHashMap<>(); + for (JsonNode plugin : json.readTree(response.body()).path("plugins")) { + String id = plugin.path("pluginId").asText(""); + String total = plugin.path("totalDownloads").asText(""); + if (!id.isEmpty() && total.matches("^\\d+$")) { + out.put(id, total); + } + } + return out; + } + catch (IOException | IllegalArgumentException e) { + log.debug("Download counts unavailable: {}", e.getMessage()); + return null; + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + + /** The CDN layout: {version}/{type}/{key}-bin.tar.gz, shared by self-hosted mirrors. */ + static String cdnPath(String key, String version) + { + String directory = "plugin"; + for (String type : List.of("executor", "convert", "fs", "parser", "scheduler")) { + if (key.startsWith("metaforge-" + type + "-")) { + directory = type; + break; + } + } + return version + "/" + directory + "/" + key + "-bin.tar.gz"; + } + + private String randomHex(int bytes) + { + byte[] value = new byte[bytes]; + random.nextBytes(value); + return hex(value); + } + + private static String hex(byte[] bytes) + { + StringBuilder out = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + out.append(Character.forDigit((b >> 4) & 0xf, 16)).append(Character.forDigit(b & 0xf, 16)); + } + return out.toString(); + } + + private static String trimSlash(String value) + { + return value == null ? "" : value.trim().replaceAll("/+$", ""); + } + + private static void deleteQuietly(Path path) + { + try { + Files.deleteIfExists(path); + } + catch (IOException e) { + log.debug("Cannot delete {}", path, e); + } + } + + private static final class CachedCount + { + private final String value; + private final long at; + + private CachedCount(String value, long at) + { + this.value = value; + this.at = at; + } + } + + private static final class CachedCatalogue + { + private final JsonNode value; + private final long at; + + private CachedCatalogue(JsonNode value, long at) + { + this.value = value; + this.at = at; + } + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiDocument.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiDocument.java new file mode 100644 index 0000000000..63b8c0d3b9 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiDocument.java @@ -0,0 +1,42 @@ +package org.devlive.metaforge.server.web; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** The web UI document (index.html) packaged with the server, read once. */ +@Slf4j +@Component +public class WebUiDocument +{ + private static final String[] LOCATIONS = {"resources/index.html", "static/index.html", "public/index.html"}; + + private volatile String html; + + public Optional html() + { + String cached = html; + if (cached != null) { + return Optional.of(cached); + } + for (String location : LOCATIONS) { + ClassPathResource resource = new ClassPathResource(location); + if (!resource.exists()) { + continue; + } + try (InputStream stream = resource.getInputStream()) { + html = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + return Optional.of(html); + } + catch (IOException e) { + log.warn("Cannot read the web UI document {}", location, e); + } + } + return Optional.empty(); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiFilter.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiFilter.java new file mode 100644 index 0000000000..345f2e4601 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/web/WebUiFilter.java @@ -0,0 +1,56 @@ +package org.devlive.metaforge.server.web; + +import org.devlive.metaforge.security.WebUiRequestMatcher; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * Answers browser page requests (/query, /admin/user ...) with the web UI document, so addresses of + * the history-routed UI open directly and survive a reload. Everything else continues as usual. + */ +@Component +public class WebUiFilter + extends OncePerRequestFilter +{ + private final WebUiRequestMatcher pages = new WebUiRequestMatcher(); + private final WebUiDocument document; + + public WebUiFilter(WebUiDocument document) + { + this.document = document; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException + { + String accept = request.getHeader(HttpHeaders.ACCEPT); + Optional html = pages.matches(request) && accept != null && accept.contains(MediaType.TEXT_HTML_VALUE) ? document.html() : Optional.empty(); + if (html.isEmpty()) { + chain.doFilter(request, response); + return; + } + byte[] body = html.get().getBytes(StandardCharsets.UTF_8); + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType(MediaType.TEXT_HTML_VALUE); + response.setCharacterEncoding(StandardCharsets.UTF_8.name()); + // The document names hashed assets; it must be revalidated so a deploy shows at once. + response.setHeader(HttpHeaders.CACHE_CONTROL, CacheControl.noCache().getHeaderValue()); + response.setContentLength(body.length); + if (!"HEAD".equals(request.getMethod())) { + response.getOutputStream().write(body); + } + } +} diff --git a/core/metaforge-server/src/main/resources/application.properties b/core/metaforge-server/src/main/resources/application.properties index 06f7f9fbb4..11b34d85a9 100644 --- a/core/metaforge-server/src/main/resources/application.properties +++ b/core/metaforge-server/src/main/resources/application.properties @@ -45,6 +45,14 @@ metaforge.audit.sql.print=false plugin.manager.extend.packages=com.fasterxml.jackson # 0 = automatic (up to 4 workers), 1 = sequential plugin.manager.load-threads=0 +# Plugin store: statistics service, download source (maven | cdn | mirror) and an optional mirror base +plugin.store.stats-url=https://forgestats.devlive.org +plugin.store.app=metaforge +plugin.store.source=maven +plugin.store.mirror= +# The store catalogue is fetched server-side and served to the browser through +# /api/v1/plugin/catalogue; empty disables the remote catalogue. +plugin.store.metadata-url=https://cdn.north.devlive.org/applications/metaforge/metadata/metadata.json ########################### Upload ########################### # Import files (SQL dumps, CSV, JSON ...) are analysed on upload. diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml new file mode 100644 index 0000000000..4501b2f1a8 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS' + SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM' + SELECT COUNT(*) FROM metaforge_role WHERE code = 'ADMIN' + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0008-user-oauth.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0008-user-oauth.xml new file mode 100644 index 0000000000..938d64ac58 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0008-user-oauth.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0009-runtime-settings-in-system-settings.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0009-runtime-settings-in-system-settings.xml new file mode 100644 index 0000000000..13b9b74c4c --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0009-runtime-settings-in-system-settings.xml @@ -0,0 +1,17 @@ + + + + + + + menu_id IN (SELECT id FROM metaforge_menu WHERE code = 'RUNTIME_CONFIGURE') + + + code = 'RUNTIME_CONFIGURE' + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0010-metaforge-executor.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0010-metaforge-executor.xml new file mode 100644 index 0000000000..98549da359 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0010-metaforge-executor.xml @@ -0,0 +1,40 @@ + + + + + + + + executor IN ('Local', 'LocalExecutor') + + + + executor IN ('Local', 'LocalExecutor') + + + + + + + + + + SELECT COUNT(*) FROM metaforge_configure WHERE category = 'EXECUTOR' AND name = 'Local' + + + + category = 'EXECUTOR' AND name = 'MetaForge' + + + + + + + category = 'EXECUTOR' AND name = 'Local' + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0011-dataset-history-metrics.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0011-dataset-history-metrics.xml new file mode 100644 index 0000000000..62498b4dbb --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0011-dataset-history-metrics.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml new file mode 100644 index 0000000000..b37e3543b7 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0013-report-share-options.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0013-report-share-options.xml new file mode 100644 index 0000000000..114dac2fd6 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0013-report-share-options.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0014-report-share-data.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0014-report-share-data.xml new file mode 100644 index 0000000000..dfc981e2ed --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0014-report-share-data.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0015-notification-content.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0015-notification-content.xml new file mode 100644 index 0000000000..55c9b66946 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0015-notification-content.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0016-user-home-page.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0016-user-home-page.xml new file mode 100644 index 0000000000..31c05e40ca --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0016-user-home-page.xml @@ -0,0 +1,14 @@ + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0017-screen-paths.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0017-screen-paths.xml new file mode 100644 index 0000000000..3baa630fcc --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0017-screen-paths.xml @@ -0,0 +1,15 @@ + + + + + + + + url = '/dashboard' + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0018-dashboard-share.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0018-dashboard-share.xml new file mode 100644 index 0000000000..80c0299a07 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0018-dashboard-share.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0019-dataset-history-checkpoint.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0019-dataset-history-checkpoint.xml new file mode 100644 index 0000000000..ebdab09ce9 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0019-dataset-history-checkpoint.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml index 118fe3fa77..26766b7efd 100644 --- a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml +++ b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml @@ -32,4 +32,17 @@ + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/seed/baseline.json b/core/metaforge-server/src/main/resources/db/seed/baseline.json index 8118733a5b..9aaba01a63 100644 --- a/core/metaforge-server/src/main/resources/db/seed/baseline.json +++ b/core/metaforge-server/src/main/resources/db/seed/baseline.json @@ -212,7 +212,7 @@ "code": "DASHBOARD", "name": "全局 - 仪表盘", "description": "全局路由:所有用户都可以访问", - "url": "/dashboard", + "url": "/screen", "group": null, "sorted": 1, "type": "VIEW", @@ -279,17 +279,17 @@ "isNew": true }, { - "code": "RUNTIME_CONFIGURE", - "name": "管理员 - 系统 - 运行时配置", + "code": "SYSTEM_SETTINGS", + "name": "管理员 - 系统 - 系统设置", "description": "管理员:管理员权限用户可以访问\\n位置:顶部管理一级子菜单", - "url": "/admin/configure", + "url": "/admin/settings", "group": null, - "sorted": 6, + "sorted": 7, "type": "VIEW", "parent": "SYSTEM", "active": true, - "i18nKey": "configure.runtime.title", - "icon": "Settings2", + "i18nKey": "navigation.systemSettings", + "icon": "SlidersHorizontal", "isNew": false }, { @@ -326,7 +326,7 @@ "DATASET", "STORE", "b6669c359a7d41e581d906446366daf3", - "RUNTIME_CONFIGURE" + "SYSTEM_SETTINGS" ], "USER": [ "HOME", diff --git a/core/metaforge-server/src/main/resources/db/seed/system-settings.json b/core/metaforge-server/src/main/resources/db/seed/system-settings.json new file mode 100644 index 0000000000..bab413d336 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/seed/system-settings.json @@ -0,0 +1,23 @@ +{ + "menus": [ + { + "code": "SYSTEM_SETTINGS", + "name": "管理员 - 系统 - 系统设置", + "description": "管理员:管理员权限用户可以访问\\n位置:顶部管理一级子菜单", + "url": "/admin/settings", + "group": null, + "sorted": 7, + "type": "VIEW", + "parent": "SYSTEM", + "active": true, + "i18nKey": "navigation.systemSettings", + "icon": "SlidersHorizontal", + "isNew": false + } + ], + "roleMenus": { + "ADMIN": [ + "SYSTEM_SETTINGS" + ] + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java new file mode 100644 index 0000000000..b6641b50cb --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java @@ -0,0 +1,275 @@ +package org.devlive.metaforge.server.auth; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.settings.FsSettings; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.service.impl.AuthServiceImpl; +import org.devlive.metaforge.service.service.impl.SystemSettingServiceImpl; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.EmailCodes; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.OAuthSettings; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.security.crypto.password.PasswordEncoder; + +import javax.mail.MessagingException; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AuthSettingsTest +{ + private static final MailSettings MAIL = MailSettings.builder().host("smtp.example.com").port(465).ssl(true).from("noreply@example.com").password("secret").build(); + + private SystemSettingService settings; + private MailService mail; + private UserService users; + private UserRepository repository; + private EmailCodes codes; + private AuthServiceImpl service; + + @BeforeEach + public void setUp() + { + settings = mock(SystemSettingService.class); + mail = mock(MailService.class); + users = mock(UserService.class); + repository = mock(UserRepository.class); + codes = new EmailCodes(); + PasswordEncoder encoder = mock(PasswordEncoder.class); + when(encoder.encode(anyString())).thenAnswer(invocation -> "hash:" + invocation.getArgument(0)); + when(repository.findByEmail(anyString())).thenReturn(Optional.empty()); + when(settings.oauth(any())).thenReturn(new OAuthSettings()); + service = new AuthServiceImpl(settings, codes, mail, users, repository, encoder, mock(InitializerConfigure.class)); + } + + private void given(AuthSettings auth, MailSettings server) + { + when(settings.auth()).thenReturn(auth); + when(settings.mail()).thenReturn(server); + } + + private static AuthSettings.AuthSettingsBuilder auth() + { + return AuthSettings.builder().registration(true).rememberDays(30); + } + + private static UserEntity user(String username, String email) + { + UserEntity user = new UserEntity(); + user.setUsername(username); + user.setEmail(email); + return user; + } + + @Test + public void emailFeaturesAreOfferedOnlyWhenTurnedOnAndMailCanBeSent() + { + given(auth().emailVerification(true).passwordReset(true).build(), new MailSettings()); + Map options = service.options(); + assertEquals(false, options.get("emailVerification")); + assertEquals(false, options.get("passwordReset")); + assertEquals(false, options.get("rememberMe")); + + given(auth().emailVerification(true).passwordReset(true).rememberMe(true).build(), MAIL); + options = service.options(); + assertEquals(true, options.get("emailVerification")); + assertEquals(true, options.get("passwordReset")); + assertEquals(true, options.get("rememberMe")); + assertFalse(service.sendCode(body("a@example.com", "unknown")).getStatus()); + } + + private static EmailCodeBody body(String email, String purpose) + { + EmailCodeBody body = new EmailCodeBody(); + body.setEmail(email); + body.setPurpose(purpose); + return body; + } + + @Test + public void signUpWithVerificationNeedsTheMailedCode() + throws MessagingException + { + given(auth().emailVerification(true).build(), MAIL); + when(users.saveOrUpdate(any(UserEntity.class))).thenReturn(CommonResponse.success(new UserEntity())); + assertTrue(service.sendCode(body("New@Example.com", "signup")).getStatus()); + ArgumentCaptor text = ArgumentCaptor.forClass(String.class); + verify(mail).send(eq(MAIL), eq("new@example.com"), anyString(), text.capture()); + String code = text.getValue().replaceAll("(?s).*?(\\d{6}).*", "$1"); + + // Asking again within a minute is refused. + assertFalse(service.sendCode(body("new@example.com", "signup")).getStatus()); + + AuthBody signup = new AuthBody(); + signup.setUsername("new_user"); + signup.setPassword("secret1"); + signup.setEmail("new@example.com"); + signup.setCode("000000".equals(code) ? "111111" : "000000"); + assertFalse(service.signup(signup).getStatus()); + signup.setCode(code); + assertTrue(service.signup(signup).getStatus()); + ArgumentCaptor saved = ArgumentCaptor.forClass(UserEntity.class); + verify(users).saveOrUpdate(saved.capture()); + assertEquals("new@example.com", saved.getValue().getEmail()); + // A code is used once. + assertFalse(service.signup(signup).getStatus()); + } + + @Test + public void signUpChecksTheSwitchAndTheUsernameRule() + { + given(auth().registration(false).build(), MAIL); + AuthBody signup = new AuthBody(); + signup.setUsername("valid_name"); + signup.setPassword("secret1"); + assertFalse(service.signup(signup).getStatus()); + + given(auth().build(), MAIL); + signup.setUsername("no spaces!"); + assertFalse(service.signup(signup).getStatus()); + verify(users, never()).saveOrUpdate(any(UserEntity.class)); + } + + @Test + public void codesAreNotMailedToUnknownAddressesOrKeptWhenSendingFails() + throws MessagingException + { + given(auth().passwordReset(true).build(), MAIL); + assertTrue(service.sendCode(body("nobody@example.com", "reset")).getStatus()); + verify(mail, never()).send(any(), anyString(), anyString(), anyString()); + + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(user("known", "known@example.com"))); + doThrow(new MessagingException("refused")).when(mail).send(any(), anyString(), anyString(), anyString()); + assertFalse(service.sendCode(body("known@example.com", "reset")).getStatus()); + assertEquals(0, codes.waiting("known@example.com", EmailCodes.Purpose.RESET)); + } + + @Test + public void aMailedCodeResetsThePassword() + { + given(auth().passwordReset(true).build(), MAIL); + UserEntity known = user("known", "known@example.com"); + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(known)); + String code = codes.issue("known@example.com", EmailCodes.Purpose.RESET); + + PasswordResetBody reset = new PasswordResetBody(); + reset.setEmail("known@example.com"); + reset.setCode(code); + reset.setPassword("short"); + assertFalse(service.reset(reset).getStatus()); + reset.setPassword("new-secret"); + assertTrue(service.reset(reset).getStatus()); + assertEquals("hash:new-secret", known.getPassword()); + verify(repository).save(known); + } + + @Test + public void signInByEmailAndRememberedSignInsLastTheSetDays() + { + given(auth().emailSignin(true).rememberMe(true).rememberDays(7).build(), new MailSettings()); + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(user("known", "known@example.com"))); + when(users.authenticate(any(UserEntity.class), anyLong())).thenReturn(CommonResponse.success("token")); + AuthBody signin = new AuthBody(); + signin.setUsername("Known@Example.com"); + signin.setPassword("secret1"); + signin.setRemember(true); + service.signin(signin); + ArgumentCaptor user = ArgumentCaptor.forClass(UserEntity.class); + verify(users).authenticate(user.capture(), eq(TimeUnit.DAYS.toMillis(7))); + assertEquals("known", user.getValue().getUsername()); + + // Without the setting, the box is ignored and the default lifetime applies. + given(auth().build(), new MailSettings()); + signin.setUsername("other"); + service.signin(signin); + verify(users).authenticate(any(UserEntity.class), eq(0L)); + } + + @Test + public void savedMailPasswordsAreKeptAndNeverSentBack() + { + RuntimeConfigureService configures = mock(RuntimeConfigureService.class); + InitializerConfigure initializer = mock(InitializerConfigure.class); + when(initializer.getRegistrationEnable()).thenReturn(true); + when(initializer.getCaptchaEnable()).thenReturn(false); + when(configures.getEffective(eq("SYSTEM"), eq("MAIL"), any())).thenReturn(MAIL.toMap()); + when(configures.getEffective(eq("SYSTEM"), eq("AUTH"), any())).thenReturn(Map.of("registration", "true")); + when(configures.save(anyString(), anyString(), anyMap(), any())).thenReturn(CommonResponse.success(null)); + SystemSettingServiceImpl system = new SystemSettingServiceImpl(configures, initializer, mail, mock(PluginManager.class)); + + MailSettings shown = (MailSettings) system.settings().get("mail"); + assertEquals("", shown.getPassword()); + assertTrue(shown.isPasswordSet()); + + assertTrue(system.save(auth().build(), MAIL.toBuilder().password("").host("smtp.other.com").build(), null).getStatus()); + @SuppressWarnings("unchecked") + ArgumentCaptor> values = ArgumentCaptor.forClass(Map.class); + verify(configures).save(eq("SYSTEM"), eq("MAIL"), values.capture(), any()); + assertEquals("secret", values.getValue().get("password")); + assertEquals("smtp.other.com", values.getValue().get("host")); + } + + @Test + public void fileStorageKeepsItsSecretAndOnlyTakesInstalledCompletePlugins() + { + RuntimeConfigureService configures = mock(RuntimeConfigureService.class); + InitializerConfigure initializer = mock(InitializerConfigure.class); + PluginManager plugins = mock(PluginManager.class); + when(plugins.getPluginInfos()).thenReturn(List.of(PluginMetadata.builder().name("LocalFs").type(PluginType.FILESYSTEM).build(), + PluginMetadata.builder().name("AmazonS3Fs").type(PluginType.FILESYSTEM).build(), + PluginMetadata.builder().name("MySQL").type(PluginType.CONNECTOR).build())); + when(configures.getEffective(anyString(), anyString(), any())).thenReturn(Map.of()); + when(configures.getEffective(eq("SYSTEM"), eq("FS"), any())).thenReturn(Map.of("type", "AmazonS3Fs", "access", "key", "secret", "stored", "endpoint", "s3.example.com", "bucket", "files")); + when(configures.save(anyString(), anyString(), anyMap(), any())).thenReturn(CommonResponse.success(null)); + SystemSettingServiceImpl system = new SystemSettingServiceImpl(configures, initializer, mail, plugins); + + FsSettings shown = (FsSettings) system.settings().get("fs"); + assertEquals("", shown.getSecret()); + assertTrue(shown.isSecretSet()); + assertEquals(List.of("AmazonS3Fs", "LocalFs"), system.settings().get("fsTypes")); + + FsSettings edited = system.fs().toBuilder().secret("").bucket("archive").build(); + assertTrue(system.save(auth().build(), new MailSettings(), null, edited).getStatus()); + @SuppressWarnings("unchecked") + ArgumentCaptor> values = ArgumentCaptor.forClass(Map.class); + verify(configures).save(eq("SYSTEM"), eq("FS"), values.capture(), any()); + assertEquals("stored", values.getValue().get("secret")); + assertEquals("archive", values.getValue().get("bucket")); + + assertFalse(system.save(auth().build(), new MailSettings(), null, FsSettings.builder().type("QiniuFs").build()).getStatus()); + assertFalse(system.save(auth().build(), new MailSettings(), null, edited.toBuilder().bucket("").build()).getStatus()); + assertTrue(system.save(auth().build(), new MailSettings(), null, FsSettings.builder().type("LocalFs").build()).getStatus()); + } + +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/OAuthSigninTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/OAuthSigninTest.java new file mode 100644 index 0000000000..a2ee1ac3b4 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/OAuthSigninTest.java @@ -0,0 +1,336 @@ +package org.devlive.metaforge.server.auth; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.entity.UserOAuthEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.UserOAuthRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.OAuthService; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.service.impl.AuthServiceImpl; +import org.devlive.metaforge.service.service.impl.OAuthServiceImpl; +import org.devlive.metaforge.service.service.impl.SystemSettingServiceImpl; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.EmailCodes; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.OAuthClient; +import org.devlive.metaforge.service.settings.OAuthProvider; +import org.devlive.metaforge.service.settings.OAuthSettings; +import org.devlive.metaforge.service.settings.OAuthStates; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.content; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +public class OAuthSigninTest +{ + private static final String SITE = "https://metaforge.example.com"; + private static final OAuthSettings GITHUB = OAuthSettings.from(OAuthProvider.GITHUB, + Map.of("enabled", "true", "clientId", "gh-client", "clientSecret", "gh-secret")); + private static final OAuthSettings OIDC = OAuthSettings.from(OAuthProvider.OIDC, Map.of("enabled", "true", "clientId", "sso-client", + "clientSecret", "sso-secret", "name", "Okta", "authorizationUrl", "https://sso.example.com/authorize", "tokenUrl", "https://sso.example.com/token", + "userinfoUrl", "https://sso.example.com/userinfo")); + + private SystemSettingService settings; + private UserService users; + private UserRepository repository; + private UserOAuthRepository links; + private MockRestServiceServer server; + private OAuthServiceImpl service; + private final Map providers = new HashMap<>(); + + @BeforeEach + public void setUp() + { + settings = mock(SystemSettingService.class); + users = mock(UserService.class); + repository = mock(UserRepository.class); + links = mock(UserOAuthRepository.class); + RestTemplate rest = new RestTemplate(); + server = MockRestServiceServer.bindTo(rest).build(); + when(settings.auth()).thenReturn(AuthSettings.builder().registration(true).siteUrl(SITE).build()); + when(settings.mail()).thenReturn(new MailSettings()); + when(settings.oauth(any())).thenAnswer(invocation -> providers.getOrDefault(invocation.getArgument(0), new OAuthSettings())); + when(repository.findByEmail(anyString())).thenReturn(Optional.empty()); + when(repository.findByUsername(anyString())).thenReturn(Optional.empty()); + when(links.findByProviderAndSubject(anyString(), anyString())).thenReturn(Optional.empty()); + when(users.issue(anyString(), anyLong())).thenAnswer(invocation -> CommonResponse.success("jwt:" + invocation.getArgument(0))); + service = new OAuthServiceImpl(settings, new OAuthStates(), new OAuthClient(rest), users, repository, links); + } + + private static UserEntity user(long id, String username, String email) + { + UserEntity user = new UserEntity(); + user.setId(id); + user.setUsername(username); + user.setEmail(email); + return user; + } + + /** The query of a location, including one after the hash of the callback page. */ + private static Map query(String location) + { + Map values = new HashMap<>(); + String query = location.substring(location.indexOf('?') + 1); + for (String pair : query.split("&")) { + String[] parts = pair.split("=", 2); + values.put(URLDecoder.decode(parts[0], StandardCharsets.UTF_8), parts.length > 1 ? URLDecoder.decode(parts[1], StandardCharsets.UTF_8) : ""); + } + return values; + } + + private void githubAnswers(String emails) + { + server.expect(requestTo("https://github.com/login/oauth/access_token")) + .andExpect(method(HttpMethod.POST)) + .andExpect(content().formDataContains(Map.of("code", "the-code", "client_id", "gh-client", "client_secret", "gh-secret", + "redirect_uri", SITE + "/api/auth/oauth/github/callback"))) + .andRespond(withSuccess("{\"access_token\":\"gh-token\",\"token_type\":\"bearer\"}", MediaType.APPLICATION_JSON)); + server.expect(requestTo("https://api.github.com/user")) + .andExpect(header("Authorization", "Bearer gh-token")) + .andRespond(withSuccess("{\"id\":42,\"login\":\"octo-cat\",\"email\":null}", MediaType.APPLICATION_JSON)); + server.expect(requestTo(OAuthProvider.GITHUB_EMAILS)) + .andRespond(withSuccess(emails, MediaType.APPLICATION_JSON)); + } + + /** Starts a sign-in and comes back with the code, as the provider would. */ + private Map signIn(String provider, String redirect) + { + OAuthService.Redirect started = service.authorize(provider, redirect, "http://ignored"); + String state = query(started.getLocation()).get("state"); + return query(service.callback(provider, "the-code", state, started.getState(), null, "http://ignored")); + } + + @Test + public void onlyProvidersThatAreOnAndConfiguredAreOffered() + { + AuthServiceImpl auth = new AuthServiceImpl(settings, new EmailCodes(), mock(MailService.class), users, repository, mock(PasswordEncoder.class), + mock(InitializerConfigure.class)); + assertEquals(List.of(), auth.options().get("providers")); + + providers.put(OAuthProvider.GITHUB, GITHUB); + providers.put(OAuthProvider.GOOGLE, GITHUB.toBuilder().enabled(false).build()); + // Turned on, but without its endpoints. + providers.put(OAuthProvider.OIDC, OIDC.toBuilder().tokenUrl("").build()); + assertEquals(List.of(Map.of("id", "github", "name", "GitHub")), auth.options().get("providers")); + + providers.put(OAuthProvider.OIDC, OIDC); + assertEquals(List.of(Map.of("id", "github", "name", "GitHub"), Map.of("id", "oidc", "name", "Okta")), auth.options().get("providers")); + } + + @Test + public void aProviderThatIsOffGoesBackToTheSignInPage() + { + OAuthService.Redirect started = service.authorize("github", "/query", null); + assertNull(started.getState()); + assertEquals(SITE + "/auth/oauth?error=disabled&redirect=%2Fquery", started.getLocation()); + assertNull(service.authorize("unknown", null, null).getState()); + } + + @Test + public void theAuthorizationRequestUsesTheSiteAddressStateAndPkce() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + OAuthService.Redirect started = service.authorize("github", "//evil.example.com", "http://internal:9096"); + assertTrue(started.getLocation().startsWith("https://github.com/login/oauth/authorize?")); + Map query = query(started.getLocation()); + assertEquals("gh-client", query.get("client_id")); + assertEquals(SITE + "/api/auth/oauth/github/callback", query.get("redirect_uri")); + assertEquals("read:user user:email", query.get("scope")); + assertEquals(started.getState(), query.get("state")); + assertEquals("S256", query.get("code_challenge_method")); + assertEquals(43, query.get("code_challenge").length()); + + // Without a site address the request's own address is used; an outside redirect is dropped. + when(settings.auth()).thenReturn(AuthSettings.builder().build()); + started = service.authorize("github", "//evil.example.com", "http://internal:9096/"); + assertEquals("http://internal:9096/api/auth/oauth/github/callback", query(started.getLocation()).get("redirect_uri")); + // A state started for one provider is refused by another. + String state = started.getState(); + assertEquals("http://internal:9096/auth/oauth?error=state", service.callback("google", "the-code", state, state, null, "http://internal:9096")); + } + + @Test + public void theStateMustComeBackOnceToTheBrowserThatStarted() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + String state = service.authorize("github", "/query", null).getState(); + assertEquals("state", query(service.callback("github", "the-code", state, "another-browser", null, null)).get("error")); + // A refused state is used up as well. + assertEquals("state", query(service.callback("github", "the-code", state, state, null, null)).get("error")); + assertEquals("state", query(service.callback("github", "the-code", "made-up", "made-up", null, null)).get("error")); + + state = service.authorize("github", "/query", null).getState(); + Map denied = query(service.callback("github", null, state, state, "access_denied", null)); + assertEquals("denied", denied.get("error")); + assertEquals("/query", denied.get("redirect")); + server.verify(); + } + + @Test + public void aVerifiedEmailLinksTheExistingUserAndTheTicketIsUsedOnce() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + when(repository.findByEmail("octo@example.com")).thenReturn(Optional.of(user(7, "octo", "octo@example.com"))); + githubAnswers("[{\"email\":\"other@example.com\",\"primary\":false,\"verified\":true},{\"email\":\"Octo@Example.com\",\"primary\":true,\"verified\":true}]"); + + Map result = signIn("github", "/query?tab=1"); + server.verify(); + assertEquals("/query?tab=1", result.get("redirect")); + ArgumentCaptor link = ArgumentCaptor.forClass(UserOAuthEntity.class); + verify(links).save(link.capture()); + assertEquals(7L, link.getValue().getUserId()); + assertEquals("github", link.getValue().getProvider()); + assertEquals("42", link.getValue().getSubject()); + verify(users, never()).saveOrUpdate(any(UserEntity.class)); + + CommonResponse token = service.redeem(result.get("ticket")); + assertTrue(token.getStatus()); + assertEquals("jwt:octo", token.getData()); + assertFalse(service.redeem(result.get("ticket")).getStatus()); + } + + @Test + public void aLinkedAccountSignsInItsUser() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + when(links.findByProviderAndSubject("github", "42")).thenReturn(Optional.of(UserOAuthEntity.builder().id(3L).userId(9L).provider("github").subject("42").build())); + when(repository.findById(9L)).thenReturn(Optional.of(user(9, "linked", null))); + githubAnswers("[]"); + + assertEquals("jwt:linked", service.redeem(signIn("github", null).get("ticket")).getData()); + verify(links, never()).save(any()); + + UserEntity disabled = user(9, "linked", null); + disabled.setActive(false); + when(repository.findById(9L)).thenReturn(Optional.of(disabled)); + server.reset(); + githubAnswers("[]"); + assertEquals("inactive", signIn("github", null).get("error")); + } + + @Test + public void aNewAccountIsCreatedOnlyWhileRegistrationIsOpen() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + when(repository.findByUsername("octo_cat")).thenReturn(Optional.of(user(1, "octo_cat", null))); + when(users.saveOrUpdate(any(UserEntity.class))).thenAnswer(invocation -> { + UserEntity saved = invocation.getArgument(0); + saved.setId(11L); + return CommonResponse.success(saved); + }); + githubAnswers("[{\"email\":\"new@example.com\",\"primary\":true,\"verified\":true}]"); + + assertEquals("jwt:", ((String) service.redeem(signIn("github", null).get("ticket")).getData()).substring(0, 4)); + ArgumentCaptor created = ArgumentCaptor.forClass(UserEntity.class); + verify(users).saveOrUpdate(created.capture()); + assertTrue(created.getValue().getUsername().matches("octo_cat_\\d{4}"), created.getValue().getUsername()); + assertEquals("new@example.com", created.getValue().getEmail()); + verify(links).save(any()); + + when(settings.auth()).thenReturn(AuthSettings.builder().registration(false).siteUrl(SITE).build()); + server.reset(); + githubAnswers("[]"); + assertEquals("registration", signIn("github", null).get("error")); + } + + @Test + public void openIdConnectUsesTheConfiguredEndpointsAndIgnoresUnverifiedEmails() + { + providers.put(OAuthProvider.OIDC, OIDC); + when(settings.auth()).thenReturn(AuthSettings.builder().registration(false).siteUrl(SITE).build()); + when(repository.findByEmail("boss@example.com")).thenReturn(Optional.of(user(1, "admin", "boss@example.com"))); + server.expect(requestTo("https://sso.example.com/token")) + .andRespond(withSuccess("{\"access_token\":\"sso-token\"}", MediaType.APPLICATION_JSON)); + server.expect(requestTo("https://sso.example.com/userinfo")) + .andExpect(header("Authorization", "Bearer sso-token")) + .andRespond(withSuccess("{\"sub\":\"abc\",\"email\":\"boss@example.com\",\"email_verified\":false}", MediaType.APPLICATION_JSON)); + + // The unverified email does not reach the administrator account, and sign-up is closed. + assertEquals("registration", signIn("oidc", null).get("error")); + server.verify(); + verify(repository, never()).findByEmail(anyString()); + verify(users, never()).issue(anyString(), anyLong()); + } + + @Test + public void aFailingProviderEndsWithAnError() + { + providers.put(OAuthProvider.GITHUB, GITHUB); + server.expect(requestTo("https://github.com/login/oauth/access_token")) + .andRespond(withSuccess("{\"error\":\"bad_verification_code\"}", MediaType.APPLICATION_JSON)); + assertEquals("failed", signIn("github", null).get("error")); + server.reset(); + server.expect(requestTo("https://github.com/login/oauth/access_token")) + .andRespond(withStatus(org.springframework.http.HttpStatus.BAD_GATEWAY)); + assertEquals("failed", signIn("github", null).get("error")); + verify(users, never()).issue(anyString(), anyLong()); + } + + @Test + public void clientSecretsAreKeptHiddenAndRequiredToTurnAProviderOn() + { + RuntimeConfigureService configures = mock(RuntimeConfigureService.class); + when(configures.getEffective(anyString(), anyString(), any())).thenReturn(Map.of()); + when(configures.getEffective(eq("SYSTEM"), eq("OAUTH_GITHUB"), any())).thenReturn(GITHUB.toMap(OAuthProvider.GITHUB)); + when(configures.save(anyString(), anyString(), anyMap(), any())).thenReturn(CommonResponse.success(null)); + SystemSettingServiceImpl system = new SystemSettingServiceImpl(configures, mock(InitializerConfigure.class), mock(MailService.class), mock(PluginManager.class)); + + @SuppressWarnings("unchecked") + Map shown = (Map) system.settings().get("oauth"); + assertEquals(List.of("github", "google", "oidc"), List.copyOf(shown.keySet())); + assertEquals("", shown.get("github").getClientSecret()); + assertTrue(shown.get("github").isSecretSet()); + + AuthSettings auth = AuthSettings.builder().build(); + // Turning on a provider without its client is refused, and nothing is saved. + assertFalse(system.save(auth, new MailSettings(), Map.of("google", OAuthSettings.builder().enabled(true).clientId("g").build())).getStatus()); + assertFalse(system.save(auth, new MailSettings(), Map.of("oidc", OIDC.toBuilder().userinfoUrl("not a url").build())).getStatus()); + assertFalse(system.save(AuthSettings.builder().siteUrl("metaforge.example.com").build(), new MailSettings(), null).getStatus()); + verify(configures, never()).save(anyString(), anyString(), anyMap(), any()); + + // An empty secret keeps the stored one, and the GitHub endpoints can not be changed. + OAuthSettings edited = shown.get("github").toBuilder().clientId("gh-new").tokenUrl("https://attacker.example.com/token").build(); + assertTrue(system.save(auth, new MailSettings(), Map.of("github", edited)).getStatus()); + @SuppressWarnings("unchecked") + ArgumentCaptor> values = ArgumentCaptor.forClass(Map.class); + verify(configures).save(eq("SYSTEM"), eq("OAUTH_GITHUB"), values.capture(), any()); + assertEquals(Map.of("enabled", "true", "clientId", "gh-new", "clientSecret", "gh-secret"), values.getValue()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/SettingTestsTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/SettingTestsTest.java new file mode 100644 index 0000000000..25b09b9d55 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/SettingTestsTest.java @@ -0,0 +1,122 @@ +package org.devlive.metaforge.server.auth; + +import org.devlive.metaforge.fs.FsRequest; +import org.devlive.metaforge.fs.FsResponse; +import org.devlive.metaforge.fs.FsService; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureField; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.impl.SystemSettingServiceImpl; +import org.devlive.metaforge.service.settings.FsSettings; +import org.devlive.metaforge.service.settings.MailService; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** The test buttons of System settings try the values in the form before anything is saved. */ +public class SettingTestsTest +{ + /** An executor whose batch size must be a number. */ + public static class ProbeExecutor + extends Plugin + { + @Override + public PluginType getType() + { + return PluginType.EXECUTOR; + } + + @Override + public List runtimeConfigures() + { + return List.of(new RuntimeConfigureField("batchSize", RuntimeConfigureFieldType.NUMBER, "1000", "Rows per batch")); + } + } + + private static SystemSettingServiceImpl service(PluginManager plugins) + { + RuntimeConfigureService configures = mock(RuntimeConfigureService.class); + when(configures.getEffective(anyString(), anyString(), any())).thenReturn(Map.of()); + InitializerConfigure initializer = mock(InitializerConfigure.class); + when(initializer.getDataHome()).thenReturn("/tmp/metaforge-data"); + return new SystemSettingServiceImpl(configures, initializer, mock(MailService.class), plugins); + } + + @Test + public void executorsCheckTheirOwnValues() + { + PluginManager plugins = mock(PluginManager.class); + when(plugins.getPluginInfos()).thenReturn(List.of(PluginMetadata.builder().name("Probe").type(PluginType.EXECUTOR).instance(new ProbeExecutor()).build())); + SystemSettingServiceImpl settings = service(plugins); + + assertTrue(settings.testRuntime("EXECUTOR", "Probe", Map.of("batchSize", "500")).getStatus()); + assertFalse(settings.testRuntime("EXECUTOR", "Probe", Map.of("batchSize", "many")).getStatus()); + assertFalse(settings.testRuntime("EXECUTOR", "Missing", Map.of()).getStatus()); + assertFalse(settings.testRuntime("UNKNOWN", "Probe", Map.of()).getStatus()); + } + + @Test + public void storageTestsWriteAndRemoveAProbeFile() + { + List calls = new ArrayList<>(); + FsService fs = new FsService() + { + @Override + public FsResponse writer(FsRequest request) + { + calls.add("write " + request.getEndpoint() + " " + request.getFileName()); + FsResponse response = new FsResponse(); + response.setSuccessful(!"broken".equals(request.getBucket())); + response.setMessage("bucket refused"); + return response; + } + + @Override + public FsResponse reader(FsRequest request) + { + return new FsResponse(); + } + + @Override + public FsResponse delete(FsRequest request) + { + calls.add("delete " + request.getFileName()); + return new FsResponse(); + } + }; + Plugin plugin = mock(Plugin.class); + when(plugin.getService(FsService.class)).thenReturn(fs); + PluginManager plugins = mock(PluginManager.class); + when(plugins.getPlugin("LocalFs")).thenReturn(Optional.of(plugin)); + when(plugins.getPlugin("AmazonS3Fs")).thenReturn(Optional.of(plugin)); + SystemSettingServiceImpl settings = service(plugins); + + assertTrue(settings.testFs(FsSettings.builder().type("LocalFs").build()).getStatus()); + assertEquals(2, calls.size()); + assertTrue(calls.get(0).startsWith("write /tmp/metaforge-data/test metaforge-test/")); + assertTrue(calls.get(1).startsWith("delete metaforge-test/")); + + FsSettings s3 = FsSettings.builder().type("AmazonS3Fs").access("key").secret("secret").endpoint("s3.example.com").bucket("broken").build(); + assertEquals("Writing a test file failed: bucket refused", settings.testFs(s3).getMessage()); + // An object store missing its bucket is refused before anything is written. + calls.clear(); + assertFalse(settings.testFs(s3.toBuilder().bucket("").build()).getStatus()); + assertTrue(calls.isEmpty()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/configure/PluginConfigurationTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/configure/PluginConfigurationTest.java index 8f43dbb1b5..a80a736369 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/configure/PluginConfigurationTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/configure/PluginConfigurationTest.java @@ -24,7 +24,7 @@ private Path manifest(String location) Objects.requireNonNull(root, "TempDir root should be injected"); Path path = root.resolve(location); Files.createDirectories(path.getParent()); - Files.writeString(path, "metaforge-executor-local=executor/metaforge-executor-local/pom.xml\n"); + Files.writeString(path, "metaforge-executor-metaforge=executor/metaforge-executor-metaforge/pom.xml\n"); return path; } diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/AdminApiSecurityTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/AdminApiSecurityTest.java index 1d531dbe32..3744db2fc6 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/AdminApiSecurityTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/AdminApiSecurityTest.java @@ -75,7 +75,7 @@ public void ordinaryUsersCannotCallAdministrativeApis() public void personalEndpointsAndEditorCompletionRemainAvailable() throws Exception { - for (String path : new String[] {"/api/v1/user", "/api/v1/user/menus", "/api/v1/user/checkUsername", "/api/v1/function/list/MySQL", "/api/v1/source/admin/count"}) { + for (String path : new String[] {"/api/v1/user", "/api/v1/user/menus", "/api/v1/user/checkUsername", "/api/v1/function/list/MySQL", "/api/v1/source/admin/count", "/api/v1/dataset/count"}) { mvc.perform(request(HttpMethod.GET, path).servletPath(path).header("X-Test-Role", "USER")).andExpect(status().isOk()); } String password = "/api/v1/user/changePassword"; @@ -97,6 +97,8 @@ public void anonymousAccessRequiresAuthentication() { String path = "/api/v1/role/list"; mvc.perform(request(HttpMethod.POST, path).servletPath(path)).andExpect(status().isUnauthorized()); + String datasetCount = "/api/v1/dataset/count"; + mvc.perform(request(HttpMethod.GET, datasetCount).servletPath(datasetCount)).andExpect(status().isUnauthorized()); } @Configuration diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/DatasetCountControllerTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/DatasetCountControllerTest.java new file mode 100644 index 0000000000..dbb54e7963 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/DatasetCountControllerTest.java @@ -0,0 +1,103 @@ +package org.devlive.metaforge.server.controller; + +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.server.support.PersistenceContext; +import org.devlive.metaforge.service.repository.DataSetColumnRepository; +import org.devlive.metaforge.service.repository.DataSetRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.FunctionService; +import org.devlive.metaforge.service.service.impl.DataSetServiceImpl; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** Counts real owned metadata rows without loading legacy identities, states or source references. */ +public class DatasetCountControllerTest +{ + private static PersistenceContext context; + private final PluginManager plugins = mock(PluginManager.class); + private MockMvc mvc; + + @BeforeAll + public static void start() throws Exception + { + context = PersistenceContext.start("h2"); + } + + @AfterAll + public static void stop() + { + context.close(); + } + + @BeforeEach + public void setup() + { + context.jdbc().update("DELETE FROM metaforge_dataset"); + // One missing code and an unreadable legacy state must not hide the correct count. + context.jdbc().update("INSERT INTO metaforge_dataset (name, code, state, source_id, user_id, active) VALUES " + + "('one', 'ONE', 'METADATA_SUCCESS', 999, 11, TRUE), " + + "('legacy', NULL, 'UNKNOWN_LEGACY_STATE', 999, 11, TRUE), " + + "('inactive', 'THREE', NULL, NULL, 11, FALSE), " + + "('other', 'OTHER', NULL, NULL, 22, TRUE), " + + "('unowned', 'UNOWNED', NULL, NULL, NULL, TRUE)"); + DataSetRepository repository = context.bean(DataSetRepository.class); + DataSetServiceImpl service = new DataSetServiceImpl(repository, mock(DataSetColumnRepository.class), null, plugins, + null, null, null, null, null, null, null, null); + mvc = MockMvcBuilders.standaloneSetup(new DataSetController(repository, service, mock(FunctionService.class), null)).build(); + signIn(11L); + } + + private static void signIn(Long id) + { + UserDetailsService user = new UserDetailsService(id, "USER", "owner", "", List.of(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(user, "", user.getAuthorities())); + } + + @AfterEach + public void clear() + { + SecurityContextHolder.clearContext(); + } + + @Test + public void countsOwnedRowsIncludingMissingCodesWithoutReadingTheStoragePlugin() throws Exception + { + mvc.perform(get("/api/v1/dataset/count").param("userId", "22")) + .andExpect(status().isOk()).andExpect(jsonPath("$.status").value(true)).andExpect(jsonPath("$.data").value(3)); + verifyNoInteractions(plugins); + } + + @Test + public void changesScopeWithTheAuthenticatedUserAndPreservesRealZero() throws Exception + { + signIn(22L); + mvc.perform(get("/api/v1/dataset/count")) + .andExpect(status().isOk()).andExpect(jsonPath("$.status").value(true)).andExpect(jsonPath("$.data").value(1)); + signIn(99L); + mvc.perform(get("/api/v1/dataset/count")) + .andExpect(status().isOk()).andExpect(jsonPath("$.status").value(true)).andExpect(jsonPath("$.data").value(0)); + } + + @Test + public void missingAuthenticationNeverCountsAllUsers() throws Exception + { + SecurityContextHolder.clearContext(); + mvc.perform(get("/api/v1/dataset/count")) + .andExpect(status().isOk()).andExpect(jsonPath("$.status").value(false)); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/PublicReportControllerTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/PublicReportControllerTest.java new file mode 100644 index 0000000000..11be6ecf61 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/controller/PublicReportControllerTest.java @@ -0,0 +1,27 @@ +package org.devlive.metaforge.server.controller; + +import org.devlive.metaforge.service.service.ReportShareService; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class PublicReportControllerTest +{ + @Test + void csvQuotesSpecialCellsAndNeutralisesFormulas() + { + ReportShareService.Table table = new ReportShareService.Table(List.of("name", "note", "amount"), List.of( + Arrays.asList("East", "=HYPERLINK(\"http://x\")", 12), + Arrays.asList("We,st", "line\nbreak", -3), + Arrays.asList(null, "@cmd", 1.5)), false); + String csv = new String(PublicReportController.csv(table), StandardCharsets.UTF_8); + assertEquals("name,note,amount\r\n" + + "East,\"'=HYPERLINK(\"\"http://x\"\")\",12\r\n" + + "\"We,st\",\"line\nbreak\",-3\r\n" + + ",'@cmd,1.5\r\n", csv); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java new file mode 100644 index 0000000000..e172f79555 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java @@ -0,0 +1,293 @@ +package org.devlive.metaforge.server.exporter; + +import org.apache.commons.io.FileUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.convert.ConvertService; +import org.devlive.metaforge.convert.model.ConvertOption; +import org.devlive.metaforge.convert.model.ConvertRequest; +import org.devlive.metaforge.convert.model.ConvertResponse; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.exporter.ExportStatements; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.devlive.metaforge.service.service.impl.DatabaseExportServiceImpl; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.ManagementCapabilities; +import org.devlive.metaforge.spi.model.ManagementCapability; +import org.devlive.metaforge.spi.model.Response; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DatabaseExportTest +{ + private final ExportStatements statements = new ExportStatements(name -> "`" + name + "`", value -> "'" + value.replace("'", "''") + "'"); + + private static ExportPlan.Filter filter(String column, String operator, String value) + { + ExportPlan.Filter filter = new ExportPlan.Filter(); + filter.setColumn(column); + filter.setOperator(operator); + filter.setValue(value); + return filter; + } + + private static ExportPlan.TablePlan table(String name, List fields, ExportPlan.Filter... filters) + { + ExportPlan.TablePlan table = new ExportPlan.TablePlan(); + table.setName(name); + table.setFields(new ArrayList<>(fields)); + table.setFilters(new ArrayList<>(List.of(filters))); + return table; + } + + @Test + public void selectsTakeFieldsFiltersAndPages() + { + ExportPlan.TablePlan chat = table("chat", List.of("id", "title"), filter("created", ">=", "2026-10-01"), filter("title", "IS NOT NULL", null)); + assertEquals("SELECT `id`, `title` FROM `app`.`chat` WHERE `created` >= '2026-10-01' AND `title` IS NOT NULL LIMIT 1000 OFFSET 2000", + statements.select("app", chat, 1000, 2000)); + assertEquals("SELECT * FROM `app`.`logs`", statements.select("app", table("logs", List.of()), 0, 0)); + assertEquals("SELECT COUNT(*) FROM `app`.`chat` WHERE `title` LIKE 'it''s%'", statements.count("app", table("chat", List.of(), filter("title", "like", "it's%")))); + } + + @Test + public void theSchemaDropsCreatesAndCanLeaveIndexesOut() + { + String create = "CREATE TABLE `chat` (\n `id` bigint NOT NULL,\n `user_id` bigint,\n PRIMARY KEY (`id`),\n KEY `idx_user` (`user_id`),\n" + + " CONSTRAINT `fk_user` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`)\n) ENGINE=InnoDB"; + ExportPlan.Structure structure = new ExportPlan.Structure(); + structure.setDrop(true); + structure.setIndexes(false); + assertEquals("DROP TABLE IF EXISTS `app`.`chat`;\nCREATE TABLE `chat` (\n `id` bigint NOT NULL,\n `user_id` bigint,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB;\n", + statements.schema("app", "chat", create, structure)); + structure.setIndexes(true); + structure.setDrop(false); + assertTrue(statements.schema("app", "chat", create, structure).contains("KEY `idx_user`")); + } + + // ---- service ---- + + private final List executed = Collections.synchronizedList(new ArrayList<>()); + + /** A format writing "headers|rows|options" as a .fake file. */ + private static final class FakeConvert + implements ConvertService + { + @Override + public List options() + { + return List.of(new ConvertOption("indent", Map.of("en", "Indent"), "2", List.of())); + } + + @Override + public String extension() + { + return "fake"; + } + + @Override + public ConvertResponse format(ConvertRequest request) + { + return new ConvertResponse(); + } + + @Override + public ConvertResponse formatStream(ConvertRequest request) + { + return new ConvertResponse(); + } + + @Override + public ConvertResponse writer(ConvertRequest request) + { + ConvertResponse response = new ConvertResponse(); + try { + FileUtils.writeStringToFile(new File(request.getPath(), request.getName() + ".fake"), request.getHeaders() + "|" + request.getColumns() + "|" + request.getOptions(), + StandardCharsets.UTF_8); + response.setSuccessful(true); + } + catch (IOException error) { + response.setSuccessful(false); + } + return response; + } + + @Override + public ConvertResponse reader(ConvertRequest request) + { + return new ConvertResponse(); + } + } + + private Response answer(String sql) + { + executed.add(sql); + List rows = new ArrayList<>(); + if (sql.startsWith("SELECT COUNT(*)")) { + rows.add(List.of(3)); + return Response.builder().isSuccessful(true).headers(List.of("COUNT(*)")).columns(rows).build(); + } + // Three rows, served a page at a time. + Matcher page = Pattern.compile("LIMIT (\\d+) OFFSET (\\d+)$").matcher(sql); + long limit = page.find() ? Long.parseLong(page.group(1)) : 3; + long offset = page.find(0) ? Long.parseLong(page.group(2)) : 0; + for (long id = offset + 1; id <= Math.min(3, offset + limit); id++) { + rows.add(List.of(id, "t" + id)); + } + return Response.builder().isSuccessful(true).headers(List.of("id", "title")).columns(rows).build(); + } + + /** chat has id and title, logs has id, listed as every plugin lists columns. */ + private static Response columns(String table) + { + List rows = new ArrayList<>(); + if ("chat".equals(table)) { + rows.add(List.of("column", "id", "bigint", "NO", "", "", "NO")); + rows.add(List.of("column", "title", "varchar(64)", "YES", "", "", "NO")); + } + else if ("logs".equals(table)) { + rows.add(List.of("column", "id", "bigint", "NO", "", "", "NO")); + } + return Response.builder().isSuccessful(true).headers(List.of("type_name", "object_name", "object_data_type", "object_nullable", "object_default_value", "object_comment", "object_autoincrement")).columns(rows).build(); + } + + private DatabaseExportService service() + { + SourceRepository repository = mock(SourceRepository.class); + PluginManager manager = mock(PluginManager.class); + Plugin source = mock(Plugin.class); + Plugin convert = mock(Plugin.class); + PluginService adapter = new PluginService() + { + @Override + public ManagementCapabilities managementCapabilities() + { + return new ManagementCapabilities("relational", ManagementCapability.keys()); + } + + @Override + public Response execute(RuntimeConfigure configure, String content) + { + return answer(content); + } + + @Override + public Response getColumns(RuntimeConfigure configure, String database, String table) + { + return columns(table); + } + + @Override + public Response getTableDdl(RuntimeConfigure configure, String database, String table, String kind) + { + return Response.builder().isSuccessful(true).content("CREATE TABLE `" + table + "` (\n `id` bigint NOT NULL,\n KEY `idx` (`id`)\n)").build(); + } + }; + SourceEntity entity = new SourceEntity(); + entity.setType("MySQL"); + when(repository.findByCode("demo")).thenReturn(Optional.of(entity)); + when(manager.getPlugin("MySQL")).thenReturn(Optional.of(source)); + when(manager.getPlugin("FakeConvert")).thenReturn(Optional.of(convert)); + when(manager.getPluginInfos()).thenReturn(List.of()); + when(convert.getService(ConvertService.class)).thenReturn(new FakeConvert()); + when(source.getService(PluginService.class)).thenReturn(adapter); + return new DatabaseExportServiceImpl(manager, repository); + } + + private static ExportPlan plan(ExportPlan.TablePlan... tables) + { + ExportPlan plan = new ExportPlan(); + plan.setTables(new ArrayList<>(List.of(tables))); + plan.setFormat("FakeConvert"); + plan.setOptions(Map.of("indent", "4")); + plan.setFileName("app export"); + plan.setBatchSize(2); + plan.getStructure().setDrop(true); + plan.getStructure().setIndexes(false); + return plan; + } + + @Test + @SuppressWarnings("unchecked") + public void theCheckFlagsUnknownFieldsAndNamesTheFiles() + { + DatabaseExportService service = service(); + CommonResponse> checked = service.check("demo", "app", plan(table("chat", List.of("id", "missing")), table("logs", List.of()), table("ghost", List.of()))); + assertTrue(checked.getStatus(), String.valueOf(checked.getMessage())); + List> tables = (List>) checked.getData().get("tables"); + assertEquals(List.of("error", "pass", "error"), tables.stream().map(row -> row.get("status")).collect(java.util.stream.Collectors.toList())); + assertEquals(List.of("Field missing does not exist"), tables.get(0).get("messages")); + assertEquals(List.of("chat.fake", "logs.fake", "ghost.fake", "schema.sql"), checked.getData().get("files")); + assertEquals("app_export.zip", checked.getData().get("output")); + assertFalse(service.execute("demo", "app", plan(table("chat", List.of("missing")))).getStatus()); + } + + @Test + public void theSampleIsWrittenByTheChosenFormatWithItsOptions() + { + CommonResponse> sample = service().sample("demo", "app", plan(table("chat", List.of("id", "title"))), "chat", 3); + assertTrue(sample.getStatus(), String.valueOf(sample.getMessage())); + assertEquals("[id, title]|[[1, t1], [2, t2], [3, t3]]|{indent=4}", sample.getData().get("content")); + assertTrue(executed.contains("SELECT `id`, `title` FROM `app`.`chat` LIMIT 3 OFFSET 0")); + } + + @Test + @SuppressWarnings("unchecked") + public void aJobReadsInPagesAndZipsEveryTableWithTheSchema() + throws Exception + { + DatabaseExportService service = service(); + CommonResponse> started = service.execute("demo", "app", plan(table("chat", List.of()), table("logs", List.of()))); + assertTrue(started.getStatus(), String.valueOf(started.getMessage())); + String id = String.valueOf(started.getData().get("jobId")); + Map job = service.job(id).getData(); + for (int attempt = 0; attempt < 100 && "running".equals(job.get("status")); attempt++) { + Thread.sleep(20); + job = service.job(id).getData(); + } + assertEquals("done", job.get("status"), String.valueOf(job.get("message"))); + assertEquals("app_export.zip", job.get("file")); + Map chat = ((List>) job.get("tables")).get(0); + assertEquals(3L, chat.get("written")); + assertTrue(executed.contains("SELECT * FROM `app`.`chat` LIMIT 2 OFFSET 2")); + + TableExportService.ExportFile file = service.file(id).getData(); + assertEquals("app_export.zip", file.getName()); + Map entries = new TreeMap<>(); + try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(file.getContent()))) { + for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) { + entries.put(entry.getName(), new String(zip.readAllBytes(), StandardCharsets.UTF_8)); + } + } + assertEquals(List.of("chat.fake", "logs.fake", "schema.sql"), new ArrayList<>(entries.keySet())); + assertEquals("[id, title]|[[1, t1], [2, t2], [3, t3]]|{indent=4}", entries.get("chat.fake")); + assertEquals("DROP TABLE IF EXISTS `app`.`chat`;\nCREATE TABLE `chat` (\n `id` bigint NOT NULL\n);\n\nDROP TABLE IF EXISTS `app`.`logs`;\nCREATE TABLE `logs` (\n `id` bigint NOT NULL\n);\n", + entries.get("schema.sql")); + assertTrue(service.job(id).getStatus(), "kept for two hours"); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/importer/ImportTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/importer/ImportTest.java index 92e711434a..4a213980be 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/importer/ImportTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/importer/ImportTest.java @@ -127,16 +127,22 @@ public Response execute(RuntimeConfigure configure, String content) { executed.add(content); List rows = new ArrayList<>(); - if (content.contains("information_schema.COLUMNS")) { - rows.add(Arrays.asList("datacap_dataset", "id", "bigint", "NO", null, "auto_increment")); - rows.add(Arrays.asList("datacap_dataset", "name", "varchar(128)", "NO", null, "")); - rows.add(Arrays.asList("datacap_dataset", "owner", "varchar(32)", "NO", null, "")); - } - else { - rows.add(List.of(2)); - } + rows.add(List.of(2)); return Response.builder().isSuccessful(true).columns(rows).build(); } + + /** datacap_dataset has an auto-increment id and two required columns; other tables do not exist. */ + @Override + public Response getColumns(RuntimeConfigure configure, String database, String table) + { + List rows = new ArrayList<>(); + if ("datacap_dataset".equals(table)) { + rows.add(Arrays.asList("column", "id", "bigint", "NO", null, "", "YES")); + rows.add(Arrays.asList("column", "name", "varchar(128)", "NO", null, "", "NO")); + rows.add(Arrays.asList("column", "owner", "varchar(32)", "NO", null, "", "NO")); + } + return Response.builder().isSuccessful(true).headers(List.of("type_name", "object_name", "object_data_type", "object_nullable", "object_default_value", "object_comment", "object_autoincrement")).columns(rows).build(); + } }; SourceEntity entity = new SourceEntity(); entity.setType("MySQL"); diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java index 8fac9a8f48..cfd1c98b4f 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java @@ -57,8 +57,11 @@ public void freshInstallCreatesSchemaAndSeedAndIsRepeatable() JdbcTemplate jdbc = jdbc(connection); assertEquals(20, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu")); + // Runtime settings live in System settings, without a menu of their own. + assertEquals(0, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu WHERE code = 'RUNTIME_CONFIGURE'")); assertEquals(4, count(jdbc, "SELECT COUNT(*) FROM metaforge_role")); assertEquals(28, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu")); + assertEquals(count(jdbc, "SELECT id FROM metaforge_menu WHERE code = 'SYSTEM'"), count(jdbc, "SELECT parent FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS'")); assertEquals(2, count(jdbc, "SELECT COUNT(*) FROM metaforge_function")); assertEquals(2, count(jdbc, "SELECT COUNT(*) FROM metaforge_scheduled")); assertEquals("系统管理员", jdbc.queryForObject("SELECT name FROM metaforge_role WHERE code = 'ADMIN'", String.class)); @@ -71,6 +74,10 @@ public void freshInstallCreatesSchemaAndSeedAndIsRepeatable() "权限-ÄÖÜ", "src-1", "MySQL", "localhost", 3306, true, false); assertEquals("权限-ÄÖÜ", jdbc.queryForObject("SELECT name FROM metaforge_source WHERE code = 'src-1'", String.class)); assertTrue(jdbc.queryForObject("SELECT is_public FROM metaforge_source WHERE code = 'src-1'", Boolean.class)); + + // new datasets default to the MetaForge executor + jdbc.update("INSERT INTO metaforge_dataset (name, code) VALUES ('orders', 'ds-1')"); + assertEquals("MetaForge", jdbc.queryForObject("SELECT executor FROM metaforge_dataset WHERE code = 'ds-1'", String.class)); } } @@ -142,7 +149,10 @@ private void assertLegacyUpgrade(Connection legacy, Connection fresh, String sch // history-owned join table: scheduled_id = history id, history_id = schedule id jdbc.update("INSERT INTO metaforge_scheduled_history_relation (scheduled_id, history_id) VALUES (91, 2)"); jdbc.update("INSERT INTO metaforge_scheduled_history_source_relation (scheduled_history_id, source_id) VALUES (91, 11)"); - jdbc.update("INSERT INTO metaforge_workflow (id, name, j_from_id, j_to_id, j_user_id, code) VALUES (101, 'flow', 11, 11, 7, 'w101')"); + jdbc.update("INSERT INTO metaforge_workflow (id, name, j_from_id, j_to_id, j_user_id, code, executor) VALUES (101, 'flow', 11, 11, 7, 'w101', 'LocalExecutor')"); + // saved Local executor settings, plus a MetaForge row seeded next to them by an interim build + jdbc.update("INSERT INTO metaforge_configure (name, category, configure) VALUES ('Local', 'EXECUTOR', '{\"batchSize\":\"5000\"}')"); + jdbc.update("INSERT INTO metaforge_configure (name, category, configure) VALUES ('MetaForge', 'EXECUTOR', '{}')"); jdbc.update("INSERT INTO metaforge_source_query (id, plugin_id, user_id, content, create_time) VALUES (111, 11, 7, 'select 1', '2026-10-28 10:11:12.345')"); DatabaseMigrator.migrate(legacy); @@ -154,7 +164,9 @@ private void assertLegacyUpgrade(Connection legacy, Connection fresh, String sch assertEquals(1, count(jdbc, "SELECT COUNT(*) FROM metaforge_user_role WHERE user_id = 7 AND role_id = 1")); assertEquals(0, count(jdbc, "SELECT COUNT(*) FROM metaforge_user_role WHERE role_id = 999")); + // the legacy menus plus System settings (core-0007), less runtime settings (core-0009) assertEquals(18, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu WHERE role_id = 1")); + assertEquals(1, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS'")); assertEquals("Redis", jdbc.queryForObject("SELECT source_type FROM metaforge_source WHERE id = 11", String.class)); assertTrue(jdbc.queryForObject("SELECT is_public FROM metaforge_source WHERE id = 11", Boolean.class)); assertEquals(7, count(jdbc, "SELECT user_id FROM metaforge_dataset WHERE id = 21")); @@ -175,6 +187,11 @@ private void assertLegacyUpgrade(Connection legacy, Connection fresh, String sch assertEquals(11, count(jdbc, "SELECT from_source_id FROM metaforge_workflow WHERE id = 101")); assertEquals(7, count(jdbc, "SELECT user_id FROM metaforge_workflow WHERE id = 101")); assertEquals(11, count(jdbc, "SELECT source_id FROM metaforge_source_query WHERE id = 111")); + // the Local executor is renamed to MetaForge, keeping its saved settings + assertEquals("MetaForge", jdbc.queryForObject("SELECT executor FROM metaforge_dataset WHERE id = 21", String.class)); + assertEquals("MetaForge", jdbc.queryForObject("SELECT executor FROM metaforge_workflow WHERE id = 101", String.class)); + assertEquals(0, count(jdbc, "SELECT COUNT(*) FROM metaforge_configure WHERE category = 'EXECUTOR' AND name = 'Local'")); + assertEquals("{\"batchSize\":\"5000\"}", jdbc.queryForObject("SELECT configure FROM metaforge_configure WHERE category = 'EXECUTOR' AND name = 'MetaForge'", String.class)); assertEquals("2026-10-28 10:11:12", jdbc.queryForObject("SELECT create_time FROM metaforge_source_query WHERE id = 111", java.sql.Timestamp.class) .toLocalDateTime().toString().replace('T', ' ')); // the navigation catalogue upgrade ran, and the seed did not duplicate the legacy catalogue diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/notify/NotificationInboxTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/notify/NotificationInboxTest.java new file mode 100644 index 0000000000..3dd1a8ab84 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/notify/NotificationInboxTest.java @@ -0,0 +1,125 @@ +package org.devlive.metaforge.server.notify; + +import liquibase.Liquibase; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; +import org.devlive.metaforge.service.notify.NotificationInbox; +import org.h2.jdbcx.JdbcDataSource; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.data.relational.core.dialect.H2Dialect; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import java.sql.Connection; +import java.sql.Timestamp; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +/** Runs the real changelog on H2, then checks the inbox only ever touches the signed-in user's rows. */ +public class NotificationInboxTest +{ + private static final long ME = 1L; + private static final long OTHER = 2L; + + private NamedParameterJdbcTemplate jdbc; + private NotificationInbox inbox; + + @BeforeEach + public void setUp() + throws Exception + { + JdbcDataSource source = new JdbcDataSource(); + source.setURL("jdbc:h2:mem:inbox-" + UUID.randomUUID() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"); + try (Connection connection = source.getConnection()) { + new Liquibase("db/changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), + DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connection))).update(""); + } + jdbc = new NamedParameterJdbcTemplate(source); + inbox = new NotificationInbox(jdbc, H2Dialect.INSTANCE); + insert(ME, "a", "DATASET_SYNC_FAILED", "数据集「订单」同步失败", "x".repeat(1500), false, "2026-11-05 10:00:00"); + insert(ME, "b", "TEST", "通知测试", "这是一条测试通知", false, "2026-11-05 11:00:00"); + insert(ME, "c", "SYNCDATA", "旧的同步通知", "历史数据 100%_done", true, "2026-11-04 09:00:00"); + insert(ME, "d", "PASSWORD_CHANGED", "登录密码已修改", "你的登录密码已修改", true, "2026-11-03 09:00:00"); + insert(OTHER, "e", "TEST", "别人的通知", "不可见", false, "2026-11-05 12:00:00"); + } + + private void insert(long user, String code, String type, String name, String content, boolean read, String time) + { + jdbc.update("INSERT INTO metaforge_notification (code, name, content, type, is_read, user_id, create_time, entity_exists) " + + "VALUES (:code, :name, :content, :type, :read, :user, :time, :exists)", + new MapSqlParameterSource("code", code).addValue("name", name).addValue("content", content).addValue("type", type) + .addValue("read", read).addValue("user", user).addValue("time", Timestamp.valueOf(time)).addValue("exists", true)); + } + + private List codes(NotificationInbox.Query query) + { + return inbox.page(ME, query).getItems().stream().map(NotificationInbox.Item::getCode).collect(Collectors.toList()); + } + + @Test + public void listsOnlyMineNewestFirstAndFiltersByStateCategoryAndWords() + { + NotificationInbox.Query all = new NotificationInbox.Query(); + Assertions.assertEquals(List.of("b", "a", "c", "d"), codes(all)); + Assertions.assertEquals(1500, inbox.page(ME, all).getItems().get(1).getContent().length()); + + NotificationInbox.Query unread = new NotificationInbox.Query(); + unread.setRead(false); + Assertions.assertEquals(List.of("b", "a"), codes(unread)); + + NotificationInbox.Query dataset = new NotificationInbox.Query(); + dataset.setCategory("DATASET"); + Assertions.assertEquals(List.of("a", "c"), codes(dataset), "old operation names count as datasets"); + + NotificationInbox.Query system = new NotificationInbox.Query(); + system.setCategory("SYSTEM"); + Assertions.assertEquals(List.of("b"), codes(system)); + + NotificationInbox.Query words = new NotificationInbox.Query(); + words.setKeyword("%_"); + Assertions.assertEquals(List.of("c"), codes(words), "wildcards are matched literally"); + + NotificationInbox.Query paged = new NotificationInbox.Query(); + paged.setSize(3); + paged.setPage(2); + Assertions.assertEquals(List.of("d"), codes(paged)); + Assertions.assertEquals(4, inbox.page(ME, paged).getTotal()); + } + + @Test + public void summaryCountsByCategory() + { + NotificationInbox.Summary summary = inbox.summary(ME); + Assertions.assertEquals(4, summary.getTotal()); + Assertions.assertEquals(2, summary.getUnread()); + Assertions.assertEquals(Long.valueOf(2), summary.getCategories().get("DATASET")); + Assertions.assertEquals(Long.valueOf(1), summary.getCategories().get("SYSTEM")); + Assertions.assertEquals(Long.valueOf(1), summary.getCategories().get("ACCOUNT")); + Assertions.assertEquals(Long.valueOf(0), summary.getCategories().get("PLUGIN")); + } + + @Test + public void readStateAndRemovalNeverReachAnotherUsersNotifications() + { + Assertions.assertFalse(inbox.mark(ME, "e", true)); + Assertions.assertFalse(inbox.delete(ME, "e")); + Assertions.assertTrue(inbox.find(ME, "e").isEmpty()); + + Assertions.assertTrue(inbox.mark(ME, "c", false)); + Assertions.assertEquals(3, inbox.unread(ME)); + Assertions.assertEquals(1, inbox.markAllRead(ME, "DATASET") - 1); + Assertions.assertEquals(1, inbox.unread(ME)); + Assertions.assertEquals(1, inbox.markAllRead(ME, null)); + Assertions.assertEquals(0, inbox.unread(ME)); + Assertions.assertEquals(1, inbox.unread(OTHER)); + + Assertions.assertTrue(inbox.delete(ME, "a")); + Assertions.assertEquals(3, inbox.deleteRead(ME)); + Assertions.assertEquals(0, inbox.summary(ME).getTotal()); + Assertions.assertEquals(1, inbox.summary(OTHER).getTotal()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/persistence/EntityMappingTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/persistence/EntityMappingTest.java index 49fe93c8f5..5bc15d9b91 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/persistence/EntityMappingTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/persistence/EntityMappingTest.java @@ -96,7 +96,7 @@ else if (!columns.contains(property.getColumnName().getReference().toLowerCase(L } }); } - assertEquals(21, entities); + assertEquals(22, entities); assertEquals(List.of(), problems); } diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardAccessTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardAccessTest.java new file mode 100644 index 0000000000..b4c249ab4e --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardAccessTest.java @@ -0,0 +1,116 @@ +package org.devlive.metaforge.server.service; + +import org.devlive.metaforge.service.entity.DashboardEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.repository.DashboardRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.repository.jdbc.LinkRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.impl.DashboardServiceImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +public class DashboardAccessTest +{ + private final DashboardRepository dashboards = mock(DashboardRepository.class); + private final ReportRepository reports = mock(ReportRepository.class); + private final LinkRepository links = mock(LinkRepository.class); + private final DashboardServiceImpl service = new DashboardServiceImpl(dashboards, reports, links); + + @BeforeEach + public void authenticate() + { + UserDetailsService user = new UserDetailsService(1L, "USER", "owner", "", List.of(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(user, "", user.getAuthorities())); + } + + @AfterEach + public void clear() + { + SecurityContextHolder.clearContext(); + } + + @Test + public void cannotOverwriteOrDeleteAnotherOwnersDashboard() + { + DashboardEntity dashboard = DashboardEntity.builder().id(5L).code("DASH").userId(2L).build(); + when(dashboards.findByCode("DASH")).thenReturn(Optional.of(dashboard)); + when(dashboards.findById(5L)).thenReturn(Optional.of(dashboard)); + + assertFalse(service.saveOrUpdate(dashboards, DashboardEntity.builder().id(5L).build()).getStatus()); + assertFalse(service.getByCode(dashboards, "DASH").getStatus()); + assertFalse(service.deleteByCode(dashboards, "DASH").getStatus()); + verify(dashboards, never()).save(any()); + verify(dashboards, never()).delete(any()); + verifyNoInteractions(reports, links); + } + + @Test + public void onlyOwnedReportsCanBeLinked() + { + ReportEntity requested = ReportEntity.builder().code("CHART").build(); + DashboardEntity draft = DashboardEntity.builder().reports(Set.of(requested)).build(); + when(reports.findByCode("CHART")).thenReturn(Optional.of(ReportEntity.builder().id(8L).userId(2L).build())); + assertFalse(service.saveOrUpdate(dashboards, draft).getStatus()); + verify(dashboards, never()).save(any()); + verifyNoInteractions(links); + + when(reports.findByCode("CHART")).thenReturn(Optional.of(ReportEntity.builder().id(9L).userId(1L).build())); + when(dashboards.save(draft)).thenReturn(DashboardEntity.builder().id(7L).build()); + assertTrue(service.saveOrUpdate(dashboards, draft).getStatus()); + verify(links).replaceDashboardReports(7L, List.of(9L)); + } + + @Test + public void onlyOwnPublishedDataScreensBecomeTheHomePage() + { + UserRepository users = mock(UserRepository.class); + service.setUsers(users); + when(dashboards.findByCode("FOREIGN")).thenReturn(Optional.of(DashboardEntity.builder().code("FOREIGN").userId(2L) + .configure("{\"version\":3,\"draft\":{},\"published\":{}}").build())); + when(dashboards.findByCode("DRAFT")).thenReturn(Optional.of(DashboardEntity.builder().code("DRAFT").userId(1L) + .configure("{\"version\":3,\"draft\":{}}").build())); + when(dashboards.findByCode("SCREEN")).thenReturn(Optional.of(DashboardEntity.builder().id(3L).code("SCREEN").userId(1L) + .configure("{\"version\":3,\"draft\":{},\"published\":{}}").build())); + + assertFalse(service.setHomePage("FOREIGN").getStatus()); + assertFalse(service.setHomePage("DRAFT").getStatus()); + assertFalse(service.setHomePage("MISSING").getStatus()); + verify(users, never()).updateHomePage(any(), any()); + + assertTrue(service.setHomePage("SCREEN").getStatus()); + verify(users).updateHomePage(1L, "SCREEN"); + assertTrue(service.setHomePage(null).getStatus()); + verify(users).updateHomePage(1L, null); + + assertTrue(service.deleteByCode(dashboards, "SCREEN").getStatus()); + verify(users).clearHomePage("SCREEN"); + } + + @Test + public void publishedLayoutsAreRecognisedInBothDocumentFormats() + { + assertTrue(DashboardServiceImpl.published("{\"version\":3,\"draft\":{},\"published\":{\"boards\":{}}}")); + assertFalse(DashboardServiceImpl.published("{\"version\":3,\"draft\":{}}")); + assertTrue(DashboardServiceImpl.published("[{\"original\":{\"code\":\"R\"}}]")); + assertFalse(DashboardServiceImpl.published(null)); + assertFalse(DashboardServiceImpl.published("not json")); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardShareTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardShareTest.java new file mode 100644 index 0000000000..ddfdf62fd5 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DashboardShareTest.java @@ -0,0 +1,368 @@ +package org.devlive.metaforge.server.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.server.support.PersistenceContext; +import org.devlive.metaforge.service.body.DashboardShareBody; +import org.devlive.metaforge.service.dashboard.share.DashboardViewerDocument; +import org.devlive.metaforge.service.entity.DashboardEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.SharePermission; +import org.devlive.metaforge.service.persistence.FilterQuery; +import org.devlive.metaforge.service.persistence.Persistence; +import org.devlive.metaforge.service.persistence.References; +import org.devlive.metaforge.service.repository.DashboardRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.RoleRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.repository.jdbc.DashboardShareRepository; +import org.devlive.metaforge.service.repository.jdbc.LinkRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.impl.DashboardServiceImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Sharing data screens: who sees what, what they may do, and what reaches them. */ +public class DashboardShareTest +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + // Owner user 2 publishes a screen with a container (row) holding a chart, and a separate table. + private static final String SCREEN = "{\"version\":3," + + "\"draft\":{\"mode\":\"wall\",\"boards\":{\"wall\":{\"widgets\":[{\"id\":\"secret\",\"reportCode\":\"DRAFT_ONLY\"}]}}}," + + "\"published\":{\"mode\":\"wall\",\"boards\":{\"wall\":{\"widgets\":[" + + "{\"id\":\"row\"},{\"id\":\"chart\",\"parentId\":\"row\",\"reportCode\":\"SALES\"},{\"id\":\"table\",\"reportCode\":\"COSTS\"}]}}}}"; + + private final DashboardRepository dashboards = mock(DashboardRepository.class); + private final ReportRepository reportRepository = mock(ReportRepository.class); + private final LinkRepository links = mock(LinkRepository.class); + private final UserRepository users = mock(UserRepository.class); + private final RoleRepository roles = mock(RoleRepository.class); + private final DashboardShareRepository shares = mock(DashboardShareRepository.class); + private final ReportService reports = mock(ReportService.class); + private final DashboardServiceImpl service = new DashboardServiceImpl(dashboards, reportRepository, links); + private final DashboardEntity screen = DashboardEntity.builder().id(9L).code("SCREEN").name("Sales").userId(2L).configure(SCREEN).build(); + + @BeforeEach + public void setup() + { + // Persistence is a static holder other tests may leave pointing at a closed database; each test gets its own. + new Persistence(mock(FilterQuery.class), mock(References.class)); + service.setUsers(users); + service.setRoles(roles); + service.setShares(shares); + service.setReports(reports); + signIn(1L); + when(dashboards.findByCode("SCREEN")).thenReturn(Optional.of(screen)); + when(dashboards.findById(9L)).thenReturn(Optional.of(screen)); + when(links.reportIdsOfDashboard(9L)).thenReturn(List.of(31L, 32L)); + when(links.roleIdsOfUser(1L)).thenReturn(List.of(7L)); + ReportEntity sales = ReportEntity.builder().id(31L).code("SALES").userId(2L).build(); + ReportEntity costs = ReportEntity.builder().id(32L).code("COSTS").userId(2L).build(); + when(reportRepository.findAllById(any())).thenReturn(List.of(sales, costs)); + when(reportRepository.findByCode("SALES")).thenReturn(Optional.of(sales)); + when(reportRepository.findByCode("COSTS")).thenReturn(Optional.of(costs)); + UserEntity owner = new UserEntity(); + owner.setId(2L); + owner.setUsername("owner"); + when(users.findById(2L)).thenReturn(Optional.of(owner)); + } + + @AfterEach + public void clear() + { + SecurityContextHolder.clearContext(); + } + + private static void signIn(Long id) + { + UserDetailsService user = new UserDetailsService(id, "USER", "user" + id, "", List.of(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(user, "", user.getAuthorities())); + } + + private void grantRole(String permission, String widgets) + { + DashboardShareRepository.Share share = new DashboardShareRepository.Share(); + share.setDashboardId(9L); + share.setWidgets(widgets); + when(shares.find(9L)).thenReturn(Optional.of(share)); + when(shares.members(9L)).thenReturn(List.of(new DashboardShareRepository.Member(DashboardShareRepository.ROLE, 7L, permission))); + } + + @Test + public void viewersGetOnlyThePublishedChosenWidgetsAndTheirReports() throws Exception + { + grantRole("VIEW", "[\"row\"]"); + CommonResponse response = service.getByCode(dashboards, "SCREEN"); + assertTrue(response.getStatus()); + DashboardEntity seen = response.getData(); + assertEquals("VIEW", seen.getPermission()); + assertEquals("owner", seen.getOwnerName()); + JsonNode document = MAPPER.readTree(seen.getConfigure()); + // The draft never reaches a viewer: both parts hold the published design. + assertEquals(document.get("published"), document.get("draft")); + assertFalse(seen.getConfigure().contains("secret")); + List ids = document.path("published").path("boards").path("wall").findValuesAsText("id"); + assertEquals(List.of("row", "chart"), ids, "the chosen container keeps its child"); + assertEquals(Set.of("SALES"), seen.getReports().stream().map(ReportEntity::getCode).collect(java.util.stream.Collectors.toSet())); + } + + @Test + public void editorsSeeTheWholeScreenAndSaveDraftsWithoutChangingOwnerOrPublishedVersion() throws Exception + { + grantRole("EDIT", "[\"row\"]"); + DashboardEntity loaded = service.getByCode(dashboards, "SCREEN").getData(); + assertEquals("EDIT", loaded.getPermission()); + assertEquals(SCREEN, loaded.getConfigure()); + + String edited = SCREEN.replace("\"mode\":\"wall\",\"boards\":{\"wall\":{\"widgets\":[{\"id\":\"secret\"", "\"mode\":\"wall\",\"boards\":{\"wall\":{\"widgets\":[{\"id\":\"draft2\"") + .replace("{\"id\":\"table\",\"reportCode\":\"COSTS\"}", "{\"id\":\"changed\"}"); + DashboardEntity draft = DashboardEntity.builder().id(9L).name("Sales").configure(edited).reports(Set.of(ReportEntity.builder().code("SALES").build())).build(); + AtomicReference saved = new AtomicReference<>(); + when(dashboards.save(any())).thenAnswer(call -> { + saved.set(call.getArgument(0)); + return call.getArgument(0); + }); + assertTrue(service.saveOrUpdate(dashboards, draft).getStatus()); + assertEquals(2L, saved.get().getUserId(), "the owner stays the owner"); + JsonNode stored = MAPPER.readTree(saved.get().getConfigure()); + assertTrue(stored.path("draft").toString().contains("draft2")); + assertTrue(stored.path("published").toString().contains("COSTS"), "editors cannot change the published screen"); + assertFalse(stored.path("published").toString().contains("changed")); + } + + @Test + public void publishersPublishWhileViewersAndStrangersCannotChangeAnything() + { + grantRole("PUBLISH", null); + when(dashboards.save(any())).thenAnswer(call -> call.getArgument(0)); + String republished = SCREEN.replace("{\"id\":\"table\",\"reportCode\":\"COSTS\"}", "{\"id\":\"changed\"}"); + DashboardEntity draft = DashboardEntity.builder().id(9L).name("Sales").configure(republished).build(); + assertTrue(service.saveOrUpdate(dashboards, draft).getData().getConfigure().contains("changed")); + + grantRole("VIEW", null); + assertFalse(service.saveOrUpdate(dashboards, DashboardEntity.builder().id(9L).configure(SCREEN).build()).getStatus()); + assertFalse(service.deleteByCode(dashboards, "SCREEN").getStatus()); + assertFalse(service.getShare("SCREEN").getStatus()); + assertFalse(service.saveShare("SCREEN", new DashboardShareBody()).getStatus()); + + when(shares.members(9L)).thenReturn(List.of()); + when(shares.find(9L)).thenReturn(Optional.empty()); + assertFalse(service.getByCode(dashboards, "SCREEN").getStatus(), "not shared at all"); + verify(dashboards, never()).delete(any()); + } + + @Test + public void viewersRefreshOnlyVisibleChartsAndTheyRunAsTheOwner() + { + grantRole("VIEW", "[\"table\"]"); + AtomicReference runAs = new AtomicReference<>(); + when(reports.refreshChart("COSTS")).thenAnswer(call -> { + runAs.set(UserDetailsService.getUser().getId()); + return CommonResponse.success(List.of(Map.of("value", 1))); + }); + assertFalse(service.refreshChart("SCREEN", "SALES").getStatus(), "a hidden widget's report stays closed"); + assertTrue(service.refreshChart("SCREEN", "COSTS").getStatus()); + assertEquals(2L, runAs.get()); + assertEquals(1L, UserDetailsService.getUser().getId(), "the viewer's identity is restored"); + } + + @Test + public void ownersShareWithExistingMembersAndPublishedWidgetsOnly() + { + signIn(2L); + when(users.existsById(5L)).thenReturn(true); + when(roles.existsById(7L)).thenReturn(true); + when(shares.find(9L)).thenReturn(Optional.empty()); + when(shares.members(9L)).thenReturn(List.of()); + + DashboardShareBody body = new DashboardShareBody(); + body.setEveryone("view"); + body.setMembers(List.of(new DashboardShareBody.Member("USER", 5L, null, "EDIT"), new DashboardShareBody.Member("ROLE", 7L, null, "PUBLISH"))); + body.setWidgets(List.of("chart", "secret")); + assertTrue(service.saveShare("SCREEN", body).getStatus(), "unknown widget ids are dropped"); + verify(shares).save(org.mockito.ArgumentMatchers.argThat(share -> "VIEW".equals(share.getEveryonePermission()) + && "[\"chart\"]".equals(share.getWidgets()) && share.getPublicToken() != null)); + verify(shares).replaceMembers(org.mockito.ArgumentMatchers.eq(9L), org.mockito.ArgumentMatchers.argThat(members -> members.size() == 2)); + + body.setWidgets(List.of("secret")); + assertFalse(service.saveShare("SCREEN", body).getStatus(), "draft-only widgets cannot be shared"); + body.setWidgets(null); + body.setMembers(List.of(new DashboardShareBody.Member("USER", 404L, null, "VIEW"))); + assertFalse(service.saveShare("SCREEN", body).getStatus()); + body.setMembers(List.of(new DashboardShareBody.Member("USER", 5L, null, "OWNER"))); + assertFalse(service.saveShare("SCREEN", body).getStatus(), "ownership is never granted"); + } + + @Test + public void viewerDocumentsDropDraftsAndUnpublishedScreens() + { + assertTrue(DashboardViewerDocument.of("{\"version\":3,\"draft\":{}}", null).isEmpty()); + assertTrue(DashboardViewerDocument.of(null, null).isEmpty()); + String whole = DashboardViewerDocument.of(SCREEN, null).orElseThrow(); + assertFalse(whole.contains("secret")); + assertEquals(Set.of("SALES", "COSTS"), DashboardViewerDocument.reportCodes(whole)); + assertEquals(Set.of("row", "chart", "table"), DashboardViewerDocument.publishedWidgetIds(SCREEN)); + String legacy = DashboardViewerDocument.of("[{\"original\":{\"code\":\"A\"}},{\"original\":{\"code\":\"B\"}}]", Set.of("legacy-1")).orElseThrow(); + assertEquals(Set.of("B"), DashboardViewerDocument.reportCodes(legacy)); + assertNull(SharePermission.grantable("OWNER").orElse(null)); + assertTrue(SharePermission.PUBLISH.atLeast(SharePermission.EDIT)); + } + + @Test + public void sharesAreStoredAndResolvedAcrossMembersRolesAndEveryone() throws Exception + { + try (PersistenceContext context = PersistenceContext.start("h2")) { + DashboardShareRepository repository = context.bean(DashboardShareRepository.class); + DashboardShareRepository.Share share = new DashboardShareRepository.Share(); + share.setDashboardId(9L); + share.setEveryonePermission("VIEW"); + share.setPublicEnabled(true); + share.setPublicToken("token-abc"); + share.setWidgets("[\"chart\"]"); + repository.save(share); + repository.replaceMembers(9L, List.of(new DashboardShareRepository.Member("ROLE", 7L, "EDIT"), + new DashboardShareRepository.Member("USER", 1L, "PUBLISH"))); + DashboardShareRepository.Share other = new DashboardShareRepository.Share(); + other.setDashboardId(10L); + repository.save(other); + repository.replaceMembers(10L, List.of(new DashboardShareRepository.Member("ROLE", 7L, "VIEW"))); + + assertEquals("[\"chart\"]", repository.find(9L).orElseThrow().getWidgets()); + assertEquals(Optional.of(9L), repository.findPublic("token-abc")); + Map mine = repository.sharedWith(1L, List.of(7L)); + assertEquals(SharePermission.PUBLISH, mine.get(9L), "the highest grant wins"); + assertEquals(SharePermission.VIEW, mine.get(10L)); + assertEquals(Map.of(9L, SharePermission.VIEW), repository.sharedWith(3L, List.of()), "everyone's grant only"); + + share.setPublicEnabled(false); + repository.save(share); + assertTrue(repository.findPublic("token-abc").isEmpty()); + repository.delete(9L); + assertTrue(repository.find(9L).isEmpty()); + assertTrue(repository.members(9L).isEmpty()); + } + } + + @Test + public void sharedScreensListTheUsersPermissionAndHideOwnScreens() + { + DashboardEntity own = DashboardEntity.builder().id(11L).code("OWN").userId(1L).configure(SCREEN).build(); + when(shares.sharedWith(anyLong(), any())).thenReturn(Map.of(9L, SharePermission.VIEW, 11L, SharePermission.EDIT)); + when(dashboards.findAllById(any())).thenReturn(List.of(screen, own)); + grantRole("VIEW", null); + List listed = service.shared().getData(); + assertEquals(1, listed.size()); + assertEquals("VIEW", listed.get(0).getPermission()); + assertEquals("owner", listed.get(0).getOwnerName()); + assertFalse(listed.get(0).getConfigure().contains("secret"), "viewers list without the draft"); + } + + @Test + public void sharedSearchAndSortOnlyExposeAuthorizedScreens() + { + DashboardEntity second = DashboardEntity.builder().id(10L).code("OTHER").name("Accounts").userId(2L).configure(SCREEN).build(); + screen.setUpdateTime(new java.util.Date(2000)); + second.setUpdateTime(new java.util.Date(1000)); + when(shares.sharedWith(anyLong(), any())).thenReturn(Map.of(9L, SharePermission.VIEW, 10L, SharePermission.EDIT)); + when(dashboards.findAllById(any())).thenReturn(List.of(screen, second)); + grantRole("VIEW", null); + assertEquals(List.of("SCREEN", "OTHER"), service.shared("", "updated").getData().stream().map(DashboardEntity::getCode).toList()); + assertEquals(List.of("OTHER", "SCREEN"), service.shared("", "name").getData().stream().map(DashboardEntity::getCode).toList()); + List filtered = service.shared(" SAL ", "updated").getData(); + assertEquals(List.of("SCREEN"), filtered.stream().map(DashboardEntity::getCode).toList()); + assertFalse(filtered.get(0).getConfigure().contains("secret"), "search never bypasses widget visibility"); + assertTrue(service.shared("not found", "updated").getData().isEmpty()); + assertFalse(service.shared("", "invalid").getStatus()); + assertFalse(service.shared("", null).getStatus()); + assertFalse(service.shared("x".repeat(201), "name").getStatus()); + } + + private DashboardShareRepository.Share publicLink(String widgets, Long expiresAt, boolean embed, String origins) + { + DashboardShareRepository.Share share = new DashboardShareRepository.Share(); + share.setDashboardId(9L); + share.setWidgets(widgets); + share.setPublicEnabled(true); + share.setPublicToken(TOKEN); + share.setPublicEmbed(embed); + share.setPublicOrigins(origins); + share.setPublicExpiresAt(expiresAt == null ? null : new java.util.Date(expiresAt)); + when(shares.findPublic(TOKEN)).thenReturn(Optional.of(9L)); + when(shares.find(9L)).thenReturn(Optional.of(share)); + screen.setActive(true); + return share; + } + + private static final String TOKEN = "abcdefghijklmnopqrstuvwxyzABCDEF"; + + @Test + @SuppressWarnings("unchecked") + public void publicLinksShowTheChosenWidgetsWithoutSqlAndExpire() + { + SecurityContextHolder.clearContext(); + ReportEntity sales = ReportEntity.builder().id(31L).code("SALES").userId(2L).query("select secret from sales").configure("{\"queryChart\":{}}").build(); + ReportEntity costs = ReportEntity.builder().id(32L).code("COSTS").userId(2L).query("select * from costs").build(); + when(reportRepository.findAllById(any())).thenReturn(List.of(sales, costs)); + publicLink("[\"row\"]", null, false, null); + + CommonResponse> view = service.publicView(TOKEN); + assertTrue(view.getStatus()); + assertFalse(view.getData().get("configure").toString().contains("secret"), "no draft"); + List> visible = (List>) view.getData().get("reports"); + assertEquals(List.of("SALES"), visible.stream().map(item -> item.get("code")).collect(java.util.stream.Collectors.toList())); + assertFalse(visible.toString().contains("select"), "no SQL reaches anonymous visitors"); + + AtomicReference runAs = new AtomicReference<>(); + when(reports.refreshChart("SALES")).thenAnswer(call -> { + runAs.set(UserDetailsService.getUser().getId()); + return CommonResponse.success(List.of()); + }); + assertTrue(service.publicRefresh(TOKEN, "SALES").getStatus()); + assertEquals(2L, runAs.get()); + assertFalse(service.publicRefresh(TOKEN, "COSTS").getStatus(), "hidden widgets stay closed"); + assertEquals(Optional.empty(), service.publicFrameAncestors(TOKEN), "not embeddable"); + + publicLink(null, System.currentTimeMillis() - 1000, true, null); + assertFalse(service.publicView(TOKEN).getStatus(), "expired"); + publicLink(null, null, true, "https://a.example.com\nhttps://b.example.com"); + assertEquals(Optional.of("'self' https://a.example.com https://b.example.com"), service.publicFrameAncestors(TOKEN)); + assertFalse(service.publicView("not-a-token").getStatus()); + publicLink(null, null, true, null).setPublicEnabled(false); + assertFalse(service.publicView(TOKEN).getStatus(), "disabled"); + } + + @Test + public void membersNeverReceiveTheOwnersSql() + { + ReportEntity sales = ReportEntity.builder().id(31L).code("SALES").userId(2L).query("select secret").shareToken("owner-token").build(); + when(reportRepository.findAllById(any())).thenReturn(List.of(sales)); + grantRole("EDIT", null); + DashboardEntity loaded = service.getByCode(dashboards, "SCREEN").getData(); + ReportEntity report = loaded.getReports().iterator().next(); + assertNull(report.getQuery()); + assertNull(report.getShareToken()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DatasetAccessTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DatasetAccessTest.java index e6fefc760d..2d65db3dfd 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DatasetAccessTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/DatasetAccessTest.java @@ -32,7 +32,7 @@ public class DatasetAccessTest private final DataSetRepository datasets = mock(DataSetRepository.class); private final DataSetColumnRepository columns = mock(DataSetColumnRepository.class); private final PluginManager plugins = mock(PluginManager.class); - private final DataSetServiceImpl service = new DataSetServiceImpl(datasets, columns, null, plugins, null, null, null, null, null, null, null); + private final DataSetServiceImpl service = new DataSetServiceImpl(datasets, columns, null, plugins, null, null, null, null, null, null, null, null); @BeforeEach public void authenticate() diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/HomeQueryActivityTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/HomeQueryActivityTest.java new file mode 100644 index 0000000000..947e855ca5 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/HomeQueryActivityTest.java @@ -0,0 +1,120 @@ +package org.devlive.metaforge.server.service; + +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.activity.QueryActivity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.PluginAuditRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.impl.PluginAuditServiceImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** The home charts use the same user, calendar window and real query records. */ +public class HomeQueryActivityTest +{ + private JdbcTemplate jdbc; + private PluginAuditServiceImpl service; + private LocalDate today; + + @BeforeEach + public void setup() + { + DriverManagerDataSource database = new DriverManagerDataSource("jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1", "sa", ""); + jdbc = new JdbcTemplate(database); + jdbc.execute("CREATE TABLE metaforge_source (id BIGINT PRIMARY KEY, name VARCHAR(100), user_id BIGINT)"); + jdbc.execute("CREATE TABLE metaforge_source_query (user_id BIGINT, source_id BIGINT, create_time TIMESTAMP)"); + jdbc.update("INSERT INTO metaforge_source VALUES (10, 'Owned', 1), (20, 'Shared', 2)"); + service = new PluginAuditServiceImpl(mock(PluginAuditRepository.class), mock(InitializerConfigure.class), mock(PluginManager.class), + new NamedParameterJdbcTemplate(database), mock(SystemSettingService.class)); + today = LocalDate.now(); + signIn(1L); + } + + @AfterEach + public void clear() + { + SecurityContextHolder.clearContext(); + jdbc.execute("SHUTDOWN"); + } + + private static void signIn(Long id) + { + UserDetailsService user = new UserDetailsService(id, "USER", "user", "", List.of(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(user, "", user.getAuthorities())); + } + + private void query(long user, long source, int daysAgo) + { + jdbc.update("INSERT INTO metaforge_source_query VALUES (?, ?, ?)", user, source, Timestamp.valueOf(today.minusDays(daysAgo).atStartOfDay())); + } + + @Test + public void bothChartsIncludeOwnQueriesOnSharedAndDeletedSourcesButExcludeOtherUsers() + { + query(1, 10, 0); + query(1, 20, 6); + query(1, 999, 2); + query(2, 10, 0); + query(2, 20, 1); + query(1, 10, 7); + query(1, 10, -1); + QueryActivity activity = service.getHomeActivity(7).getData(); + assertEquals(7, activity.getDays()); + assertEquals(3, activity.getTotal()); + assertEquals(7, activity.getTrend().size()); + assertEquals(today.minusDays(6).toString(), activity.getTrend().get(0).getDate()); + assertEquals(today.toString(), activity.getTrend().get(6).getDate()); + assertEquals(0, activity.getTrend().get(1).getCount(), "missing days stay visible as zero"); + assertEquals(1, activity.getTrend().get(0).getCount(), "the inclusive start boundary"); + assertEquals(3, activity.getTrend().stream().mapToLong(QueryActivity.Day::getCount).sum()); + assertEquals(3, activity.getSources().stream().mapToLong(QueryActivity.Source::getCount).sum()); + assertEquals(List.of("", "Owned", "Shared"), activity.getSources().stream().map(QueryActivity.Source::getName).toList()); + } + + @Test + public void thirtyDaysHasTheSameBoundariesAndCountsAsItsDistribution() + { + query(1, 10, 0); + query(1, 10, 29); + query(1, 20, 8); + query(1, 20, 30); + QueryActivity activity = service.getHomeActivity(30).getData(); + assertEquals(30, activity.getTrend().size()); + assertEquals(3, activity.getTotal()); + assertEquals("Owned", activity.getSources().get(0).getName()); + assertEquals(2, activity.getSources().get(0).getCount()); + assertEquals(3, activity.getTrend().stream().mapToLong(QueryActivity.Day::getCount).sum()); + assertEquals(3, activity.getSources().stream().mapToLong(QueryActivity.Source::getCount).sum()); + assertEquals(1, service.getHomeActivity(7).getData().getTotal()); + } + + @Test + public void emptyAndUnsupportedPeriodsNeverInventData() + { + QueryActivity activity = service.getHomeActivity(7).getData(); + assertEquals(0, activity.getTotal()); + assertTrue(activity.getSources().isEmpty()); + assertTrue(activity.getTrend().stream().allMatch(day -> day.getCount() == 0)); + assertFalse(service.getHomeActivity(365).getStatus()); + assertFalse(service.getHomeActivity(-1).getStatus()); + signIn(null); + assertFalse(service.getHomeActivity(7).getStatus()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/MenuAccessRepositoryTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/MenuAccessRepositoryTest.java index 3e58deab4a..6a28037f2e 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/MenuAccessRepositoryTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/MenuAccessRepositoryTest.java @@ -111,7 +111,7 @@ public void userNavigationUsesSqlUnionsAndCompletesOnlyRequiredAncestors() { UserRepository users = mock(UserRepository.class); RoleRepository roles = mock(RoleRepository.class); - UserServiceImpl service = new UserServiceImpl(users, roles, null, menus, null, null, null, null, null, null, null, null, null, null, null, null); + UserServiceImpl service = new UserServiceImpl(users, roles, null, menus, null, null, null, null, null, null, null, null, null, null, null, null, null, null); UserDetailsService principal = new UserDetailsService(10L, "OWNER", "owner", "", List.of(), null); SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); List tree = service.getMenus().getData(); diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginReferencesTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginReferencesTest.java new file mode 100644 index 0000000000..7edf61c9c8 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginReferencesTest.java @@ -0,0 +1,58 @@ +package org.devlive.metaforge.server.service; + +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType.H2; + +public class PluginReferencesTest +{ + @Test + public void countsTrackedReferencesAndGuardsUninstall() + { + var database = new EmbeddedDatabaseBuilder().generateUniqueName(true).setType(H2).build(); + try { + var jdbc = new JdbcTemplate(database); + jdbc.execute("CREATE TABLE metaforge_source (name VARCHAR(64), source_type VARCHAR(64))"); + jdbc.execute("CREATE TABLE metaforge_dataset (name VARCHAR(64), executor VARCHAR(64), scheduler VARCHAR(64))"); + jdbc.execute("CREATE TABLE metaforge_workflow (name VARCHAR(64), executor VARCHAR(64))"); + jdbc.update("INSERT INTO metaforge_source VALUES (?, ?)", "Production", "MYSQL"); + jdbc.update("INSERT INTO metaforge_dataset VALUES (?, ?, ?)", "Daily dataset", "Local", "LocalScheduler"); + jdbc.update("INSERT INTO metaforge_workflow VALUES (?, ?)", "Daily workflow", "LocalExecutor"); + var plugins = List.of(plugin("MySQL", PluginType.CONNECTOR), plugin("Redis", PluginType.CONNECTOR), + plugin("MetaForge", PluginType.EXECUTOR), plugin("LocalScheduler", PluginType.SCHEDULER), plugin("JsonConvert", PluginType.CONVERT)); + + var service = new PluginReferences(jdbc); + var listed = service.list(plugins); + assertEquals("Production", listed.get("MySQL").get(0).getName()); + assertEquals(2, listed.get("MetaForge").size()); + assertEquals("dataset", listed.get("LocalScheduler").get(0).getKind()); + assertTrue(listed.get("Redis").isEmpty()); + assertFalse(listed.containsKey("JsonConvert")); + + assertTrue(service.isInUse("MySQL", plugins)); + assertTrue(service.isInUse("MetaForge", plugins)); + assertTrue(service.isInUse("LocalScheduler", plugins)); + assertFalse(service.isInUse("Redis", plugins)); + + jdbc.update("DELETE FROM metaforge_source"); + assertFalse(service.isInUse("MySQL", plugins)); + } + finally { + database.shutdown(); + } + } + + private static PluginMetadata plugin(String name, PluginType type) + { + return PluginMetadata.builder().name(name).type(type).build(); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginStoreTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginStoreTest.java new file mode 100644 index 0000000000..d7536f28c5 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/PluginStoreTest.java @@ -0,0 +1,248 @@ +package org.devlive.metaforge.server.service; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PluginStoreTest +{ + private static final byte[] PACKAGE = "metaforge plugin archive".getBytes(StandardCharsets.UTF_8); + private static final String CATALOGUE = "{\"key\":\"plugin\",\"children\":[{\"key\":\"metaforge-plugin-mysql\",\"label\":\"MySQL\"}]}"; + + private HttpServer server; + private String base; + private final List requests = new CopyOnWriteArrayList<>(); + private final AtomicReference statsSha = new AtomicReference<>(); + private volatile int statsStatus = 302; + + @BeforeEach + void start() + throws Exception + { + statsSha.set(sha(PACKAGE)); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + base = "http://127.0.0.1:" + server.getAddress().getPort(); + server.createContext("/", exchange -> { + String path = exchange.getRequestURI().toString(); + requests.add(exchange.getRequestMethod() + " " + path); + if (path.startsWith("/v1/apps/metaforge/plugins/")) { + if (statsStatus != 302) { + exchange.sendResponseHeaders(statsStatus, -1); + exchange.close(); + return; + } + String source = path.contains("source=cdn") ? "cdn" : "maven"; + exchange.getResponseHeaders().add("Location", base + "/" + source + "/pkg.tar.gz"); + exchange.getResponseHeaders().add("X-Forge-Attempt-Id", exchange.getRequestHeaders().getFirst("Idempotency-Key")); + exchange.getResponseHeaders().add("X-Forge-Receipt", "signed-receipt"); + exchange.getResponseHeaders().add("X-Forge-Sha256", statsSha.get()); + exchange.getResponseHeaders().add("X-Forge-Size", String.valueOf(PACKAGE.length)); + exchange.sendResponseHeaders(302, -1); + } + else if (path.endsWith("/complete")) { + byte[] body = "{\"status\":\"confirmed\"}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + else if (path.startsWith("/v1/apps/metaforge/stats")) { + byte[] body = "{\"plugins\":[{\"pluginId\":\"metaforge-plugin-mysql\",\"totalDownloads\":\"9007199254740993\"}]}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + else if (path.equals("/catalogue.json")) { + byte[] body = CATALOGUE.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + else if (path.equals("/catalogue-broken.json")) { + byte[] body = "not json at all".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + else if (path.endsWith(".tar.gz")) { + exchange.sendResponseHeaders(200, PACKAGE.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(PACKAGE); + } + } + else { + exchange.sendResponseHeaders(404, -1); + } + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() + { + server.stop(0); + } + + private PluginStore store(String source, String mirror) + { + return store(source, mirror, base + "/catalogue.json"); + } + + private PluginStore store(String source, String mirror, String metadataUrl) + { + return new PluginStore(base, "metaforge", source, mirror, metadataUrl, HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build(), false); + } + + private static PluginStore.Package target() + { + return PluginStore.Package.builder().key("metaforge-plugin-mysql").version("2026.0.0").url("https://unused.invalid/x.tar.gz").build(); + } + + private static String sha(byte[] bytes) + throws Exception + { + StringBuilder out = new StringBuilder(); + for (byte b : MessageDigest.getInstance("SHA-256").digest(bytes)) { + out.append(String.format("%02x", b)); + } + return out.toString(); + } + + @Test + void countedInstallVerifiesThenReports() + throws IOException + { + AtomicReference installed = new AtomicReference<>(); + assertTrue(store("cdn", "").install(target(), (archive, name) -> { + assertEquals("metaforge-plugin-mysql", name); + assertTrue(archive.getFileName().toString().endsWith("-bin.tar.gz"), "the installer detects archives by extension"); + installed.set(read(archive)); + return true; + })); + assertArrayEquals(PACKAGE, installed.get()); + assertTrue(requests.stream().anyMatch(r -> r.contains("download?source=cdn"))); + assertTrue(requests.contains("GET /cdn/pkg.tar.gz")); + assertTrue(requests.stream().anyMatch(r -> r.startsWith("POST /v1/apps/metaforge/downloads/") && r.endsWith("/complete"))); + } + + @Test + void aFailedInstallIsNotReported() + throws IOException + { + assertFalse(store("maven", "").install(target(), (archive, name) -> false)); + assertTrue(requests.stream().noneMatch(r -> r.endsWith("/complete"))); + } + + @Test + void aTamperedPackageIsRejectedOnEverySource() + { + statsSha.set("0".repeat(64)); + assertThrows(PluginStore.StoreException.class, () -> store("maven", "").install(target(), (archive, name) -> true)); + assertTrue(requests.contains("GET /maven/pkg.tar.gz")); + assertTrue(requests.contains("GET /cdn/pkg.tar.gz"), "the next source is tried before giving up"); + assertTrue(requests.stream().noneMatch(r -> r.endsWith("/complete"))); + } + + @Test + void aMirrorUsesTheReceiptButDownloadsFromTheMirror() + throws IOException + { + assertTrue(store("mirror", base + "/mirror/").install(target(), (archive, name) -> true)); + assertTrue(requests.contains("GET /mirror/2026.0.0/plugin/metaforge-plugin-mysql-bin.tar.gz")); + assertTrue(requests.stream().noneMatch(r -> r.startsWith("GET /maven/") || r.startsWith("GET /cdn/"))); + assertTrue(requests.stream().anyMatch(r -> r.endsWith("/complete"))); + } + + @Test + void anUnknownVersionInstallsDirectlyWithoutCounting() + throws Exception + { + statsStatus = 404; + PluginStore.Package direct = PluginStore.Package.builder().key("metaforge-plugin-mysql").version("2026.0.0") + .url(base + "/direct/pkg.tar.gz").sha256(sha(PACKAGE)).size((long) PACKAGE.length).build(); + assertTrue(store("maven", "").install(direct, (archive, name) -> true)); + assertTrue(requests.contains("GET /direct/pkg.tar.gz")); + assertTrue(requests.stream().noneMatch(r -> r.endsWith("/complete"))); + PluginStore.Package wrong = PluginStore.Package.builder().key("metaforge-plugin-mysql").version("2026.0.0") + .url(base + "/direct/pkg.tar.gz").sha256("1".repeat(64)).build(); + assertThrows(PluginStore.StoreException.class, () -> store("maven", "").install(wrong, (archive, name) -> true)); + } + + @Test + void anUnreachableServiceStillInstalls() + throws IOException + { + PluginStore offline = new PluginStore("http://127.0.0.1:1", "metaforge", "maven", "", "", HttpClient.newHttpClient(), false); + PluginStore.Package direct = PluginStore.Package.builder().key("metaforge-plugin-mysql").version("2026.0.0").url(base + "/direct/pkg.tar.gz").build(); + assertTrue(offline.install(direct, (archive, name) -> true)); + assertEquals(Map.of(), offline.downloads(List.of("metaforge-plugin-mysql"))); + } + + @Test + void downloadCountsStayExactAndCached() + { + PluginStore store = store("maven", ""); + Map counts = store.downloads(List.of("metaforge-plugin-mysql", "metaforge-plugin-unknown", "BAD KEY")); + assertEquals(Map.of("metaforge-plugin-mysql", "9007199254740993"), counts); + store.downloads(List.of("metaforge-plugin-mysql", "metaforge-plugin-unknown")); + assertEquals(1, requests.stream().filter(r -> r.startsWith("GET /v1/apps/metaforge/stats")).count()); + } + + @Test + void springCreatesTheStoreFromItsConfiguration() + { + try (org.springframework.context.annotation.AnnotationConfigApplicationContext context = new org.springframework.context.annotation.AnnotationConfigApplicationContext()) { + context.registerBean(org.springframework.context.support.PropertySourcesPlaceholderConfigurer.class); + context.registerBean(PluginStore.class); + context.refresh(); + assertTrue(context.getBean(PluginStore.class).downloads(List.of("BAD KEY")).isEmpty()); + } + } + + @Test + void theCatalogueIsFetchedCachedAndValidated() + { + PluginStore store = store("maven", ""); + assertEquals("MySQL", store.catalogue().get("children").get(0).get("label").asText()); + assertEquals("MySQL", store.catalogue().get("children").get(0).get("label").asText()); + assertEquals(1, requests.stream().filter(r -> r.equals("GET /catalogue.json")).count(), "the catalogue is cached briefly"); + assertNull(store("maven", "", base + "/catalogue-broken.json").catalogue(), "a non-JSON catalogue is refused"); + assertNull(new PluginStore(base, "metaforge", "maven", "", "", HttpClient.newHttpClient(), false).catalogue(), "an empty URL disables the catalogue"); + } + + @Test + void invalidRequestsAreRefusedBeforeAnyDownload() + { + PluginStore store = store("maven", ""); + assertThrows(PluginStore.StoreException.class, () -> store.install(PluginStore.Package.builder().key("../etc").version("1").build(), (a, n) -> true)); + assertThrows(PluginStore.StoreException.class, () -> store.install(PluginStore.Package.builder().key("metaforge-plugin-mysql").version("1/../x").build(), (a, n) -> true)); + assertTrue(requests.isEmpty()); + } + + private static byte[] read(Path path) + { + try { + return Files.readAllBytes(path); + } + catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/SyncJobManagerTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/SyncJobManagerTest.java new file mode 100644 index 0000000000..2483c0f4dd --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/SyncJobManagerTest.java @@ -0,0 +1,370 @@ +package org.devlive.metaforge.server.service; + +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; +import org.devlive.metaforge.service.dataset.sync.SyncJob; +import org.devlive.metaforge.service.dataset.sync.SyncJobManager; +import org.devlive.metaforge.service.dataset.sync.SyncMetricsView; +import org.devlive.metaforge.service.dataset.sync.SyncTransition; +import org.devlive.metaforge.service.entity.DatasetHistoryEntity; +import org.devlive.metaforge.service.repository.DatasetHistoryRepository; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SyncJobManagerTest +{ + private final Map rows = new ConcurrentHashMap<>(); + private final AtomicLong ids = new AtomicLong(); + private final AtomicInteger saves = new AtomicInteger(); + private SyncJobManager manager; + + @BeforeEach + public void setUp() + { + DatasetHistoryRepository repository = mock(DatasetHistoryRepository.class); + when(repository.save(any(DatasetHistoryEntity.class))).thenAnswer(invocation -> { + DatasetHistoryEntity history = invocation.getArgument(0); + if (history.getId() == null) { + history.setId(ids.incrementAndGet()); + } + rows.put(history.getId(), history); + saves.incrementAndGet(); + return history; + }); + when(repository.findById(any())).thenAnswer(invocation -> Optional.ofNullable(rows.get((Long) invocation.getArgument(0)))); + RuntimeConfigureService settings = mock(RuntimeConfigureService.class); + when(settings.getEffective(anyString(), anyString(), anyList())) + .thenAnswer(invocation -> new java.util.HashMap<>(Map.of(SyncJobManager.MAX_CONCURRENT_SYNCS, "1"))); + manager = new SyncJobManager(repository, settings); + } + + @AfterEach + public void tearDown() + { + manager.shutdown(); + } + + private DatasetHistoryEntity created(long datasetId) + { + DatasetHistoryEntity history = new DatasetHistoryEntity(); + history.setDatasetId(datasetId); + history.setState(RunState.CREATED); + history.setAttempt(1); + return history; + } + + private Optional submit(long datasetId, Consumer body) + { + return manager.submit(datasetId, () -> { + return saveNew(created(datasetId)); + }, body); + } + + private DatasetHistoryEntity saveNew(DatasetHistoryEntity history) + { + history.setId(ids.incrementAndGet()); + rows.put(history.getId(), history); + return history; + } + + private static void await(CountDownLatch latch) + { + try { + assertTrue(latch.await(10, TimeUnit.SECONDS), "timed out"); + } + catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + + private void awaitState(Long historyId, RunState state) + { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline) { + if (rows.get(historyId).getState() == state) { + return; + } + Thread.onSpinWait(); + } + assertEquals(state, rows.get(historyId).getState()); + } + + @Test + public void aDatasetRunsOneSyncAtATimeAndIsFreedWhenItEnds() + { + CountDownLatch release = new CountDownLatch(1); + SyncJob first = submit(7, job -> { + await(release); + job.transition(RunState.SUCCESS, null); + }).orElseThrow(); + AtomicBoolean factoryCalled = new AtomicBoolean(); + assertTrue(manager.submit(7L, () -> { + factoryCalled.set(true); + return saveNew(created(7)); + }, job -> {}).isEmpty()); + assertFalse(factoryCalled.get(), "a rejected submit must not create a history row"); + assertTrue(manager.isActive(7L)); + + release.countDown(); + awaitState(first.getHistoryId(), RunState.SUCCESS); + long deadline = System.currentTimeMillis() + 10_000; + while (manager.isActive(7L) && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + assertFalse(manager.isActive(7L)); + assertEquals(List.of(RunState.CREATED, RunState.QUEUE, RunState.RUNNING, RunState.SUCCESS), + manager.view(first.getHistoryId()).orElseThrow().getTransitions().stream().map(SyncTransition::getTo).collect(Collectors.toList())); + } + + @Test + public void syncsBeyondTheLimitWaitInTheQueueInOrder() + { + CountDownLatch release = new CountDownLatch(1); + List started = new java.util.concurrent.CopyOnWriteArrayList<>(); + SyncJob a = submit(1, job -> { + started.add(1L); + await(release); + job.transition(RunState.SUCCESS, null); + }).orElseThrow(); + awaitState(a.getHistoryId(), RunState.RUNNING); + SyncJob b = submit(2, job -> { + started.add(2L); + job.transition(RunState.SUCCESS, null); + }).orElseThrow(); + SyncJob c = submit(3, job -> { + started.add(3L); + job.transition(RunState.SUCCESS, null); + }).orElseThrow(); + assertEquals(RunState.QUEUE, b.getState()); + assertEquals(RunState.QUEUE, c.getState()); + + release.countDown(); + awaitState(c.getHistoryId(), RunState.SUCCESS); + assertEquals(List.of(1L, 2L, 3L), started); + } + + @Test + public void aQueuedSyncIsCancelledWithoutRunning() + { + CountDownLatch release = new CountDownLatch(1); + SyncJob blocker = submit(1, job -> { + await(release); + job.transition(RunState.SUCCESS, null); + }).orElseThrow(); + awaitState(blocker.getHistoryId(), RunState.RUNNING); + AtomicBoolean ran = new AtomicBoolean(); + SyncJob queued = submit(2, job -> ran.set(true)).orElseThrow(); + + assertTrue(manager.cancelQueued(queued)); + assertEquals(RunState.STOPPED, rows.get(queued.getHistoryId()).getState()); + assertFalse(manager.isActive(2L), "cancelling frees the dataset at once"); + assertFalse(manager.cancelQueued(blocker), "a running sync is not cancelled here"); + + release.countDown(); + awaitState(blocker.getHistoryId(), RunState.SUCCESS); + assertFalse(ran.get()); + } + + @Test + public void illegalTransitionsAreRejectedAndUnfinishedBodiesFail() + { + SyncJob job = submit(5, running -> { + assertFalse(running.transition(RunState.QUEUE, null), "RUNNING cannot go back to QUEUE"); + assertTrue(running.transition(RunState.STOPPING, null)); + assertFalse(running.transition(RunState.RUNNING, null)); + throw new IllegalStateException("boom"); + }).orElseThrow(); + awaitState(job.getHistoryId(), RunState.FAILURE); + assertEquals("boom", rows.get(job.getHistoryId()).getMessage()); + + SyncJob silent = submit(6, running -> {}).orElseThrow(); + awaitState(silent.getHistoryId(), RunState.FAILURE); + assertEquals("The sync ended without a final state", rows.get(silent.getHistoryId()).getMessage()); + } + + @Test + public void metricsAreThrottledAndTheFinalSnapshotIsPersisted() + { + CountDownLatch done = new CountDownLatch(1); + SyncJob job = submit(8, running -> { + int before = saves.get(); + assertTrue(manager.onMetrics(running, metrics(10, 100), false)); + assertFalse(manager.onMetrics(running, metrics(20, 100), false), "a second report within the interval is not saved"); + assertEquals(before + 1, saves.get()); + assertEquals(20L, manager.view(running.getHistoryId()).orElseThrow().getMetrics().getRowsWritten(), "the live view has the latest report"); + assertTrue(manager.onMetrics(running, metrics(100, 100), true)); + running.transition(RunState.SUCCESS, null); + done.countDown(); + }).orElseThrow(); + await(done); + awaitState(job.getHistoryId(), RunState.SUCCESS); + long deadline = System.currentTimeMillis() + 10_000; + while (manager.find(job.getHistoryId()).isPresent() && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + + DatasetHistoryEntity row = rows.get(job.getHistoryId()); + assertEquals(100L, row.getRowsWritten()); + assertEquals(100L, row.getProcessedCount()); + assertEquals(100L, row.getTotalCount()); + SyncMetricsView view = manager.view(job.getHistoryId()).orElseThrow(); + assertFalse(view.isLive()); + assertEquals(100L, view.getMetrics().getRowsWritten()); + assertEquals(4, view.getTransitions().size()); + } + + private void awaitFinished(SyncJob job) + { + long deadline = System.currentTimeMillis() + 10_000; + while (manager.find(job.getHistoryId()).isPresent() && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + assertTrue(manager.find(job.getHistoryId()).isEmpty()); + } + + @Test + public void aPausedSyncKeepsItsDatasetAndRunsAgainWhenResumed() + { + AtomicInteger runs = new AtomicInteger(); + SyncJob job = submit(9, running -> { + if (runs.incrementAndGet() == 1) { + running.update(history -> history.setCheckpoint("at-500")); + running.transition(RunState.PAUSED, null); + } + else { + assertEquals("at-500", running.getCheckpoint()); + running.transition(RunState.SUCCESS, null); + } + }).orElseThrow(); + awaitState(job.getHistoryId(), RunState.PAUSED); + long deadline = System.currentTimeMillis() + 10_000; + while (runs.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + + assertTrue(manager.isActive(9L), "a paused sync still holds the dataset"); + assertTrue(manager.active().stream().anyMatch(active -> active.getHistoryId().equals(job.getHistoryId())), "and is listed as unfinished"); + assertTrue(manager.find(job.getHistoryId()).isPresent()); + assertTrue(submit(9, running -> {}).isEmpty()); + + assertTrue(manager.resume(job)); + awaitState(job.getHistoryId(), RunState.SUCCESS); + awaitFinished(job); + assertEquals(2, runs.get()); + // jobs 和 activeByDataset 在收尾时先后清理,两个 map 的清空之间有观察窗口 + // jobs and activeByDataset are cleared one after the other; poll for the release + long released = System.currentTimeMillis() + 10_000; + while (manager.isActive(9L) && System.currentTimeMillis() < released) { + Thread.onSpinWait(); + } + assertFalse(manager.isActive(9L), "the finished sync must release the dataset"); + assertFalse(manager.resume(job), "only a paused sync can resume"); + } + + @Test + public void aPausedSyncCanBeCancelledAndCanThenRunAgain() + { + SyncJob job = submit(10, running -> { + running.update(history -> history.setCheckpoint("at-500")); + running.transition(RunState.PAUSED, null); + }).orElseThrow(); + awaitState(job.getHistoryId(), RunState.PAUSED); + // 工作线程退出后再取消,模拟用户稍后操作 / cancel after the worker has left, as a user would + long deadline = System.currentTimeMillis() + 10_000; + while (!job.getState().equals(RunState.PAUSED) && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + + assertTrue(manager.cancelQueued(job)); + DatasetHistoryEntity row = rows.get(job.getHistoryId()); + assertEquals(RunState.STOPPED, row.getState()); + assertEquals("Cancelled while paused", row.getMessage()); + assertFalse(manager.isActive(10L)); + assertTrue(SyncJobManager.canRetry(row)); + + AtomicBoolean ran = new AtomicBoolean(); + SyncJob again = manager.retry(row, running -> { + ran.set(true); + running.transition(RunState.SUCCESS, null); + }).orElseThrow(); + awaitState(again.getHistoryId(), RunState.SUCCESS); + awaitFinished(again); + assertTrue(ran.get()); + assertEquals(2, row.getAttempt()); + assertEquals(List.of(RunState.CREATED, RunState.QUEUE, RunState.RUNNING, RunState.PAUSED, RunState.STOPPED, RunState.QUEUE, RunState.RUNNING, RunState.SUCCESS), + manager.view(row.getId()).orElseThrow().getTransitions().stream().map(SyncTransition::getTo).collect(Collectors.toList())); + } + + @Test + public void onlyRunsThatWriteNothingTwiceMayRunAgain() + { + DatasetHistoryEntity row = created(11); + row.setState(RunState.SUCCESS); + row.setCheckpoint("at-500"); + assertFalse(SyncJobManager.canRetry(row), "a successful run is final"); + + row.setState(RunState.RUNNING); + assertFalse(SyncJobManager.canRetry(row), "a running sync is not finished"); + + for (RunState state : List.of(RunState.FAILURE, RunState.STOPPED, RunState.TIMEOUT, RunState.INTERRUPTED)) { + row.setState(state); + row.setCheckpoint("at-500"); + assertTrue(SyncJobManager.canRetry(row), state + " with a checkpoint"); + row.setCheckpoint(null); + row.setRowsWritten(500L); + assertFalse(SyncJobManager.canRetry(row), state + " that wrote rows without a checkpoint"); + row.setRowsWritten(0L); + assertEquals(state != RunState.INTERRUPTED, SyncJobManager.canRetry(row), state + " that wrote nothing"); + } + } + + @Test + public void aDatasetWithAnUnfinishedSyncRefusesToRunAnotherAgain() + { + CountDownLatch release = new CountDownLatch(1); + SyncJob active = submit(12, running -> { + await(release); + running.transition(RunState.SUCCESS, null); + }).orElseThrow(); + DatasetHistoryEntity finished = created(12); + finished.setId(ids.incrementAndGet()); + finished.setState(RunState.STOPPED); + finished.setCheckpoint("at-500"); + assertTrue(manager.retry(finished, running -> {}).isEmpty()); + release.countDown(); + awaitState(active.getHistoryId(), RunState.SUCCESS); + } + + private static ExecutorMetrics metrics(long written, long total) + { + ExecutorMetrics metrics = new ExecutorMetrics(); + metrics.setRowsRead(written); + metrics.setRowsWritten(written); + metrics.setTotalRows(total); + metrics.setProgress(written * 100.0 / total); + return metrics; + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableCompareServiceTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableCompareServiceTest.java index eff72b85fe..832e091ff2 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableCompareServiceTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableCompareServiceTest.java @@ -39,19 +39,7 @@ private Response answer(String content) sql.add(content); ObjectMapper mapper = new ObjectMapper(); List rows = new ArrayList<>(); - if (content.contains("information_schema.COLUMNS")) { - if (content.contains("TABLE_SCHEMA = 'app'")) { - rows.add(Arrays.asList("id", "bigint", "NO", null, "主键")); - rows.add(Arrays.asList("name", "varchar(100)", "NO", null, "")); - rows.add(Arrays.asList("status", "tinyint", "NO", "1", "")); - } - else { - rows.add(Arrays.asList("id", "bigint", "NO", null, "主键")); - rows.add(Arrays.asList("name", "varchar(100)", "YES", null, "")); - rows.add(Arrays.asList("is_deleted", "tinyint", "NO", "0", "")); - } - } - else if (content.contains("KEY_COLUMN_USAGE")) { + if (content.contains("KEY_COLUMN_USAGE")) { rows.add(mapper.createObjectNode().put("object_name", "id")); } else if (content.startsWith("SHOW CREATE TABLE")) { @@ -69,6 +57,26 @@ else if (content.contains("FROM `bak`.`report`")) { return Response.builder().isSuccessful(true).columns(rows).build(); } + /** The plugin's column listing, with the standard headers every plugin returns. */ + private static Response columns(String database) + { + List rows = new ArrayList<>(); + if ("app".equals(database)) { + rows.add(Arrays.asList("column", "id", "bigint", "NO", null, "主键")); + rows.add(Arrays.asList("column", "name", "varchar(100)", "NO", null, "")); + rows.add(Arrays.asList("column", "status", "tinyint", "NO", "1", "")); + rows.add(Arrays.asList("index", "PRIMARY", "", "", null, "")); + } + else { + rows.add(Arrays.asList("column", "id", "bigint", "NO", null, "主键")); + rows.add(Arrays.asList("column", "name", "varchar(100)", "YES", null, "")); + rows.add(Arrays.asList("column", "is_deleted", "tinyint", "NO", "0", "")); + } + return Response.builder().isSuccessful(true) + .headers(Arrays.asList("type_name", "object_name", "object_data_type", "object_nullable", "object_default_value", "object_comment")) + .columns(rows).build(); + } + private TableCompareService service() { SourceRepository repository = mock(SourceRepository.class); @@ -87,6 +95,12 @@ public Response execute(RuntimeConfigure configure, String content) { return answer(content); } + + @Override + public Response getColumns(RuntimeConfigure configure, String database, String table) + { + return columns(database); + } }; SourceEntity entity = new SourceEntity(); entity.setType("MySQL"); diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableExportServiceTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableExportServiceTest.java index 788964a6b7..b7feb40c1a 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableExportServiceTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/TableExportServiceTest.java @@ -85,10 +85,6 @@ private Response answer(String content) rows.add(List.of(3215L)); return Response.builder().isSuccessful(true).headers(List.of("COUNT(*)")).columns(rows).build(); } - if (content.contains("information_schema.COLUMNS")) { - rows.add(List.of("id", "bigint")); - return Response.builder().isSuccessful(true).headers(List.of("COLUMN_NAME", "COLUMN_TYPE")).columns(rows).build(); - } rows.add(new ArrayList<>(Arrays.asList(1, null))); return Response.builder().isSuccessful(true).headers(List.of("id", "name")).columns(rows).build(); } @@ -112,6 +108,13 @@ public Response execute(RuntimeConfigure configure, String content) { return answer(content); } + + @Override + public Response getColumns(RuntimeConfigure configure, String database, String table) + { + return Response.builder().isSuccessful(true).headers(List.of("type_name", "object_name", "object_data_type", "object_nullable", "object_default_value", "object_comment", "object_autoincrement")) + .columns(new ArrayList<>(List.of(List.of("column", "id", "bigint", "NO", "", "", "NO")))).build(); + } }; SourceEntity entity = new SourceEntity(); entity.setType("MySQL"); diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/WorkflowExecutionTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/WorkflowExecutionTest.java new file mode 100644 index 0000000000..9e38838d7e --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/service/WorkflowExecutionTest.java @@ -0,0 +1,464 @@ +package org.devlive.metaforge.server.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.executor.ExecutorService; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; +import org.devlive.metaforge.executor.configure.ExecutorRequest; +import org.devlive.metaforge.executor.configure.ExecutorResponse; +import org.devlive.metaforge.executor.workflow.WorkflowPlan; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.server.support.PersistenceContext; +import org.devlive.metaforge.service.body.FilterBody; +import org.devlive.metaforge.service.body.FilterCriterionBody; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.entity.WorkflowEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.itransient.configuration.Configuration; +import org.devlive.metaforge.service.itransient.configuration.ConnectionConfiguration; +import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.repository.WorkflowRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.impl.WorkflowServiceImpl; +import org.devlive.metaforge.service.workflow.WorkflowCatalog; +import org.devlive.metaforge.service.workflow.WorkflowRun; +import org.devlive.metaforge.spi.PluginService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class WorkflowExecutionTest +{ + @TempDir + Path home; + private final WorkflowRepository repository = mock(WorkflowRepository.class); + private final SourceRepository sources = mock(SourceRepository.class); + private final PluginManager plugins = mock(PluginManager.class); + private final RuntimeConfigureService runtime = mock(RuntimeConfigureService.class); + private final ExecutorService executor = mock(ExecutorService.class); + private final ExecutorService seatunnel = mock(ExecutorService.class); + private WorkflowServiceImpl service; + private WorkflowCatalog catalog; + + @BeforeEach + public void setup() throws Exception + { + UserDetailsService user = new UserDetailsService(1L, "USER", "owner", "", List.of(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(user, "", user.getAuthorities())); + InitializerConfigure initializer = mock(InitializerConfigure.class); + when(initializer.getDataHome()).thenReturn(home.toString()); + engine("MetaForge", executor, "metaforge-executor-metaforge"); + engine("Seatunnel", seatunnel, "metaforge-executor-seatunnel"); + when(plugins.getPluginInfos()).thenReturn(List.of( + PluginMetadata.builder().name("Seatunnel").type(PluginType.EXECUTOR).build(), + PluginMetadata.builder().name("MetaForge").type(PluginType.EXECUTOR).build(), + PluginMetadata.builder().name("MockJdbc").type(PluginType.CONNECTOR).build())); + when(runtime.getEffective(anyString(), anyString(), any())).thenReturn(Map.of("batchSize", "1000")); + // Compilation belongs to each executor and is tested with it; here it only has to reach the executor. + when(executor.compile(any())).thenAnswer(call -> request(call.getArgument(0))); + when(seatunnel.compile(any())).thenAnswer(call -> request(call.getArgument(0))); + catalog = new WorkflowCatalog(plugins); + service = new WorkflowServiceImpl(initializer, plugins, repository, sources, runtime, catalog); + when(repository.save(any())).thenAnswer(call -> call.getArgument(0)); + } + + /** Registers an executor plugin serving the declaration that ships in the executor module. */ + private void engine(String name, ExecutorService service, String module) throws Exception + { + Plugin plugin = mock(Plugin.class); + when(plugin.getName()).thenReturn(name); + when(plugin.getType()).thenReturn(PluginType.EXECUTOR); + when(plugin.getService(ExecutorService.class)).thenReturn(service); + when(plugin.runtimeConfigures()).thenReturn(List.of()); + when(plugins.getPlugin(name)).thenReturn(Optional.of(plugin)); + Path declaration = Path.of("../../executor", module, "src/main/resources/org/devlive/metaforge/executor/workflow/workflow.yaml"); + when(service.workflowDeclaration()).thenReturn(declaration.toUri().toURL()); + } + + private static ExecutorRequest request(WorkflowPlan plan) + { + ExecutorRequest request = new ExecutorRequest(plan.getTaskName(), plan.getUserName(), + plan.source().getSources().isEmpty() ? new org.devlive.metaforge.executor.configure.ExecutorConfigure("Jdbc") : plan.source().source("source"), + plan.sink().getSources().isEmpty() ? new org.devlive.metaforge.executor.configure.ExecutorConfigure("Jdbc") : plan.sink().source("source")); + request.setWorkHome(plan.getWorkHome()); + request.setOptions(plan.getOptions()); + return request; + } + + @AfterEach + public void close() + { + service.close(); + SecurityContextHolder.clearContext(); + } + + private WorkflowEntity workflow() + { + NodeConfiguration input = new NodeConfiguration(); + input.setId(UUID.randomUUID().toString()); + input.setTid("metaforge-source"); + input.setKey("Source"); + input.setCategory("source"); + input.setData(Map.of("source", "SOURCE", "query", "select id from events")); + NodeConfiguration output = new NodeConfiguration(); + output.setId(UUID.randomUUID().toString()); + output.setTid("metaforge-sink"); + output.setKey("Sink"); + output.setCategory("sink"); + output.setData(Map.of("source", "TARGET", "database", "warehouse", "table", "events", "columns", Map.of("id", "event_id"))); + Configuration graph = new Configuration(); + graph.setNodes(List.of(input, output)); + graph.setConnections(List.of(new ConnectionConfiguration("connection", input.getId() + "-output", output.getId() + "-input"))); + WorkflowEntity workflow = new WorkflowEntity(); + workflow.setCode("WORKFLOW"); + workflow.setName("Workflow"); + workflow.setExecutor("MetaForge"); + workflow.setUserId(1L); + workflow.setConfigure(graph); + return workflow; + } + + @Test + public void catalogAndGraphValidationFollowTheExecutor() + { + assertEquals(List.of("MetaForge", "Seatunnel"), catalog.executors().stream().map(item -> item.definition().getName()).collect(Collectors.toList())); + assertEquals(List.of("metaforge-sink", "metaforge-source"), catalog.definitions("MetaForge").stream().map(NodeConfiguration::getId).sorted().collect(Collectors.toList())); + assertTrue(catalog.definitions("Seatunnel").stream().noneMatch(node -> node.getId().startsWith("metaforge"))); + assertTrue(catalog.executor("MetaForge").definition().getCapabilities().cooperativeStop()); + assertFalse(catalog.executor("Seatunnel").definition().getCapabilities().isRowCount()); + assertEquals(0, catalog.executor("MetaForge").definition().limit("transform")); + assertEquals(3, catalog.executor("Seatunnel").definition().limit("transform")); + assertTrue(catalog.executor("MetaForge").definition().getMetrics().containsAll(List.of("rowsWritten", "series", "bottleneck"))); + assertTrue(catalog.executor("Seatunnel").definition().getMetrics().isEmpty()); + WorkflowEntity workflow = workflow(); + assertTrue(catalog.validate(workflow).isEmpty()); + workflow.getConfigure().getConnections().get(0).setTarget(workflow.getConfigure().getNodes().get(0).getId() + "-input"); + assertFalse(catalog.validate(workflow).isEmpty()); + workflow = workflow(); + workflow.getConfigure().getNodes().get(0).setTid("sql-transform"); + assertFalse(catalog.validate(workflow).isEmpty()); + workflow = workflow(); + workflow.getConfigure().setSettings(Map.of("batchSize", 0)); + assertTrue(catalog.validate(workflow).stream().anyMatch(error -> error.contains("out of range"))); + workflow.getConfigure().setSettings(Map.of("batchSize", 200)); + assertTrue(catalog.validate(workflow).isEmpty()); + } + + @Test + public void executorSpecificValidationRunsAfterTheGenericChecks() + { + WorkflowEntity workflow = workflow(); + when(executor.validate(any())).thenReturn(List.of("Executor rejected the pipeline")); + assertEquals(List.of("Executor rejected the pipeline"), catalog.validate(workflow)); + WorkflowEntity broken = workflow(); + broken.getConfigure().setConnections(List.of()); + assertFalse(catalog.validate(broken).contains("Executor rejected the pipeline")); + } + + @Test + public void savingADraftDoesNotRunItAndPublishingOnlyValidates() + { + WorkflowEntity workflow = workflow(); + when(repository.findByCode("WORKFLOW")).thenReturn(Optional.of(workflow)); + assertTrue(service.saveOrUpdate(repository, workflow).getStatus()); + assertEquals(RunState.CREATED, workflow.getState()); + assertTrue(service.publish("WORKFLOW").getStatus()); + assertNotNull(workflow.getConfigure().getPublishedAt()); + assertEquals(1, workflow.getConfigure().getRevision()); + verify(executor, never()).start(any()); + } + + @Test + public void foreignWorkflowsCannotBeReadChangedExecutedOrDeleted() + { + WorkflowEntity foreign = workflow(); + foreign.setUserId(2L); + when(repository.findByCode("WORKFLOW")).thenReturn(Optional.of(foreign)); + assertFalse(service.getByCode(repository, "WORKFLOW").getStatus()); + assertFalse(service.saveOrUpdate(repository, workflow()).getStatus()); + assertFalse(service.restart("WORKFLOW").getStatus()); + assertFalse(service.publish("WORKFLOW").getStatus()); + assertFalse(service.runs("WORKFLOW").getStatus()); + assertFalse(service.log("WORKFLOW").getStatus()); + assertFalse(service.stop("WORKFLOW").getStatus()); + assertFalse(service.deleteByCode(repository, "WORKFLOW").getStatus()); + verify(repository, never()).save(any()); + verify(repository, never()).delete(any()); + } + + @Test + public void executionUsesSavedSourcesAndPersistsRealRunResults() throws Exception + { + WorkflowEntity workflow = workflow(); + when(repository.findByCode("WORKFLOW")).thenReturn(Optional.of(workflow)); + Plugin connector = mock(Plugin.class); + PluginService driver = mock(PluginService.class); + when(connector.getService(PluginService.class)).thenReturn(driver); + when(driver.supportsStreaming()).thenReturn(true); + when(plugins.getPlugin("MockJdbc")).thenReturn(Optional.of(connector)); + for (String code : List.of("SOURCE", "TARGET")) { + SourceEntity source = new SourceEntity(); + source.setCode(code); source.setUserId(1L); source.setType("MockJdbc"); + when(sources.findByCode(code)).thenReturn(Optional.of(source)); + } + AtomicReference submitted = new AtomicReference<>(); + CountDownLatch completed = new CountDownLatch(1); + when(executor.start(any())).thenAnswer(call -> { + submitted.set(call.getArgument(0)); + ExecutorResponse response = new ExecutorResponse(false, true, RunState.SUCCESS, null); + response.setCount(7); + ExecutorMetrics metrics = new ExecutorMetrics(); + metrics.setRowsWritten(7); + response.setMetrics(metrics); + response.setPreview(List.of(Map.of("event_id", 3))); + completed.countDown(); + return response; + }); + CommonResponse started = service.restart("WORKFLOW"); + assertTrue(started.getStatus(), String.valueOf(started.getMessage())); + assertTrue(completed.await(3, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + WorkflowRun run; + do { + run = service.run("WORKFLOW", started.getData()).getData(); + if (run.getState() != RunState.SUCCESS) { Thread.sleep(5); } + } while (run.getState() != RunState.SUCCESS && System.nanoTime() < deadline); + assertEquals(RunState.SUCCESS, run.getState()); + assertEquals(7, run.getCount()); + assertEquals(3, run.getPreview().get(0).get("event_id")); + verify(executor).compile(org.mockito.ArgumentMatchers.argThat(plan -> plan.getTaskName().equals(started.getData()) + && plan.getNodes().size() == 2 && "source".equals(plan.getNodes().get(0).getCategory().getValue()) + && plan.source().getSources().containsKey("source") && plan.sink().getSources().containsKey("source") + && "warehouse".equals(plan.sink().getValues().get("database")) + && "1000".equals(plan.getOptions().get("batchSize")))); + assertEquals(started.getData(), submitted.get().getTaskName()); + assertNull(service.runs("WORKFLOW").getData().get(0).getPreview()); + Path persisted = Path.of(workflow.getWork(), "run-" + started.getData() + ".json"); + // The service flips the in-memory state before it writes the run file, so the + // file can lag the SUCCESS state and the poll has to tolerate a missing file. + WorkflowRun disk = null; + do { + try { + disk = new ObjectMapper().readValue(persisted.toFile(), WorkflowRun.class); + if (disk.getState() == RunState.SUCCESS) { break; } + } + catch (java.io.FileNotFoundException missing) { + // Not written yet; retry within the deadline. + } + Thread.sleep(5); + } while (System.nanoTime() < deadline); + assertTrue(disk != null && disk.getState() == RunState.SUCCESS, "the run file was not persisted in time"); + assertEquals(7, disk.getMetrics().getRowsWritten()); + assertEquals(3, disk.getPreview().get(0).get("event_id")); + assertFalse(service.log("WORKFLOW", "../../secrets").getStatus()); + verify(sources).findByCode(eq("SOURCE")); + } + @Test + public void stoppingIsCooperativeAndPreventsAnOverlappingRestart() throws Exception + { + WorkflowEntity workflow = workflow(); + when(repository.findByCode("WORKFLOW")).thenReturn(Optional.of(workflow)); + Plugin connector = mock(Plugin.class); + PluginService driver = mock(PluginService.class); + when(connector.getService(PluginService.class)).thenReturn(driver); + when(driver.supportsStreaming()).thenReturn(true); + when(plugins.getPlugin("MockJdbc")).thenReturn(Optional.of(connector)); + for (String code : List.of("SOURCE", "TARGET")) { + SourceEntity source = new SourceEntity(); + source.setCode(code); source.setUserId(1L); source.setType("MockJdbc"); + when(sources.findByCode(code)).thenReturn(Optional.of(source)); + } + CountDownLatch running = new CountDownLatch(1); + CountDownLatch exiting = new CountDownLatch(1); + when(executor.start(any())).thenAnswer(call -> { + running.countDown(); + assertTrue(exiting.await(3, TimeUnit.SECONDS)); + return new ExecutorResponse(false, true, RunState.SUCCESS, null); + }); + when(executor.stop(any())).thenReturn(new ExecutorResponse(false, true, RunState.STOPPED, null)); + CommonResponse started = service.restart("WORKFLOW"); + assertTrue(started.getStatus()); + try { + assertTrue(running.await(3, TimeUnit.SECONDS)); + assertTrue(service.stop("WORKFLOW").getStatus()); + assertEquals(RunState.STOPPED, workflow.getState()); + assertFalse(service.restart("WORKFLOW").getStatus()); + assertEquals(RunState.STOPPED, service.run("WORKFLOW", started.getData()).getData().getState()); + } + finally { + exiting.countDown(); + } + } + + @Test + public void missingExternalEngineCountsRemainUnknown() throws Exception + { + WorkflowEntity workflow = workflow(); + workflow.setExecutor("Seatunnel"); + for (NodeConfiguration node : workflow.getConfigure().getNodes()) { + node.setTid("source".equals(node.getCategory()) ? "jdbc-source" : "jdbc-sink"); + node.setKey("Jdbc"); + node.setData(Map.of("url", "jdbc:example:test", "driver", "com.mysql.jdbc.Driver")); + } + when(repository.findByCode("WORKFLOW")).thenReturn(Optional.of(workflow)); + when(seatunnel.start(any())).thenReturn(new ExecutorResponse(false, true, RunState.SUCCESS, null)); + CommonResponse started = service.restart("WORKFLOW"); + assertTrue(started.getStatus(), String.valueOf(started.getMessage())); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + WorkflowRun run; + do { + run = service.run("WORKFLOW", started.getData()).getData(); + if (run.getState() != RunState.SUCCESS) { Thread.sleep(5); } + } while (run.getState() != RunState.SUCCESS && System.nanoTime() < deadline); + assertEquals(RunState.SUCCESS, run.getState()); + assertNull(run.getCount()); + verify(executor, never()).start(any()); + } + + @Test + public void savedWorkflowCanBeReopenedFromJdbc() throws Exception + { + try (PersistenceContext context = PersistenceContext.start("h2")) { + WorkflowRepository realRepository = context.bean(WorkflowRepository.class); + InitializerConfigure initializer = mock(InitializerConfigure.class); + when(initializer.getDataHome()).thenReturn(home.toString()); + WorkflowServiceImpl realService = new WorkflowServiceImpl(initializer, plugins, realRepository, sources, runtime, catalog); + try { + WorkflowEntity draft = workflow(); + draft.getConfigure().setDescription("Persisted workflow description"); + draft.getConfigure().setTimeout(120L); + CommonResponse saved = realService.saveOrUpdate(realRepository, draft); + assertTrue(saved.getStatus(), String.valueOf(saved.getMessage())); + assertTrue(realService.publish("WORKFLOW").getStatus()); + WorkflowEntity reopened = realService.getByCode(realRepository, "WORKFLOW").getData(); + assertEquals(RunState.CREATED, reopened.getState()); + assertEquals("Persisted workflow description", reopened.getConfigure().getDescription()); + assertEquals(120L, reopened.getConfigure().getTimeout()); + assertEquals(1, reopened.getConfigure().getRevision()); + assertNotNull(reopened.getConfigure().getPublishedAt()); + assertEquals("event_id", ((Map) reopened.getConfigure().getNodes().get(1).getData().get("columns")).get("id")); + assertEquals(1L, realService.getAll(realRepository, new FilterBody()).getData().getTotal()); + FilterBody filter = new FilterBody(); + filter.setConditions(Map.of("state", new FilterCriterionBody("FAILURE", true))); + assertEquals(0L, realService.getAll(realRepository, filter).getData().getTotal()); + filter.setConditions(Map.of("executor", new FilterCriterionBody("MetaForge", true))); + assertEquals(1L, realService.getAll(realRepository, filter).getData().getTotal()); + verify(executor, never()).start(any()); + } + finally { + realService.close(); + } + } + } + + @Test + public void queryColumnsComeFromThePluginSchemaWithoutReadingRows() + { + Plugin connector = mock(Plugin.class); + PluginService driver = mock(PluginService.class); + when(connector.getService(PluginService.class)).thenReturn(driver); + when(driver.supportsStreaming()).thenReturn(true); + when(plugins.getPlugin("MockJdbc")).thenReturn(Optional.of(connector)); + SourceEntity source = new SourceEntity(); + source.setCode("SOURCE"); source.setUserId(1L); source.setType("MockJdbc"); + when(sources.findByCode("SOURCE")).thenReturn(Optional.of(source)); + AtomicReference statement = new AtomicReference<>(); + List rowsRead = new java.util.ArrayList<>(); + org.mockito.Mockito.doAnswer(call -> { + statement.set(call.getArgument(1)); + org.devlive.metaforge.spi.adapter.RowCallback callback = call.getArgument(3); + callback.onSchema(List.of("id", "name"), List.of("BIGINT", "VARCHAR")); + rowsRead.add("row"); + callback.onRow(List.of(1, "a")); + return null; + }).when(driver).executeStream(any(), anyString(), org.mockito.ArgumentMatchers.anyInt(), any()); + CommonResponse>> columns = service.queryColumns("SOURCE", "select id, name from events;"); + assertTrue(columns.getStatus(), String.valueOf(columns.getMessage())); + assertEquals(List.of(Map.of("name", "id", "type", "BIGINT"), Map.of("name", "name", "type", "VARCHAR")), columns.getData()); + assertEquals("select id, name from events", statement.get()); + assertTrue(rowsRead.isEmpty()); + assertFalse(service.queryColumns("SOURCE", " ").getStatus()); + SourceEntity foreign = new SourceEntity(); + foreign.setCode("FOREIGN"); foreign.setUserId(2L); foreign.setType("MockJdbc"); + when(sources.findByCode("FOREIGN")).thenReturn(Optional.of(foreign)); + assertFalse(service.queryColumns("FOREIGN", "select 1").getStatus()); + } + @Test + @SuppressWarnings("unchecked") + public void queryPreviewStopsAfterTheRowLimit() + { + Plugin connector = mock(Plugin.class); + PluginService driver = mock(PluginService.class); + when(connector.getService(PluginService.class)).thenReturn(driver); + when(driver.supportsStreaming()).thenReturn(true); + when(plugins.getPlugin("MockJdbc")).thenReturn(Optional.of(connector)); + SourceEntity source = new SourceEntity(); + source.setCode("SOURCE"); source.setUserId(1L); source.setType("MockJdbc"); + when(sources.findByCode("SOURCE")).thenReturn(Optional.of(source)); + java.util.concurrent.atomic.AtomicInteger delivered = new java.util.concurrent.atomic.AtomicInteger(); + org.mockito.Mockito.doAnswer(call -> { + org.devlive.metaforge.spi.adapter.RowCallback callback = call.getArgument(3); + callback.onSchema(List.of("id", "note"), List.of("BIGINT", "TEXT")); + for (int index = 0; index < 500; index++) { + delivered.incrementAndGet(); + callback.onRow(java.util.Arrays.asList(index, index == 0 ? "x".repeat(5000) : null)); + } + return null; + }).when(driver).executeStream(any(), anyString(), org.mockito.ArgumentMatchers.anyInt(), any()); + CommonResponse> preview = service.queryPreview("SOURCE", "select id, note from events"); + assertTrue(preview.getStatus(), String.valueOf(preview.getMessage())); + List> rows = (List>) preview.getData().get("rows"); + assertEquals(50, rows.size()); + assertEquals(50, delivered.get()); + assertEquals(0, rows.get(0).get("id")); + assertEquals(1001, rows.get(0).get("note").toString().length()); + assertNull(rows.get(1).get("note")); + assertEquals(2, ((List) preview.getData().get("columns")).size()); + } + @Test + public void runTimesAreFormattedByTheServer() throws Exception + { + WorkflowRun run = new WorkflowRun(); + run.setId(UUID.randomUUID().toString()); + run.setStartedAt(1762354581000L); + ObjectMapper mapper = new ObjectMapper().setTimeZone(java.util.TimeZone.getTimeZone("GMT+8")); + String json = mapper.writeValueAsString(run); + assertTrue(json.contains("\"startTime\":\"2025-11-05 22:56:21\""), json); + assertTrue(json.contains("\"finishTime\":null"), json); + // Stored records with or without the derived fields read back. + assertEquals(1762354581000L, mapper.readValue(json, WorkflowRun.class).getStartedAt()); + assertEquals(1762354581000L, mapper.readValue("{\"startedAt\":1762354581000}", WorkflowRun.class).getStartedAt()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/support/PersistenceContext.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/support/PersistenceContext.java index 66b4fc80fd..dbf20a4e07 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/support/PersistenceContext.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/support/PersistenceContext.java @@ -5,6 +5,7 @@ import org.devlive.metaforge.service.persistence.FilterQuery; import org.devlive.metaforge.service.persistence.Persistence; import org.devlive.metaforge.service.persistence.References; +import org.devlive.metaforge.service.repository.jdbc.DashboardShareRepository; import org.devlive.metaforge.service.repository.jdbc.LinkRepository; import org.devlive.metaforge.service.repository.jdbc.MenuAccessRepository; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -76,7 +77,8 @@ public void close() } @Configuration - @Import({DataJdbcConfigure.class, FilterQuery.class, References.class, Persistence.class, LinkRepository.class, MenuAccessRepository.class}) + @Import({DataJdbcConfigure.class, FilterQuery.class, References.class, Persistence.class, LinkRepository.class, MenuAccessRepository.class, + DashboardShareRepository.class}) static class Beans { @Bean diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/web/WebUiFilterTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/web/WebUiFilterTest.java new file mode 100644 index 0000000000..615c4217bb --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/web/WebUiFilterTest.java @@ -0,0 +1,86 @@ +package org.devlive.metaforge.server.web; + +import org.devlive.metaforge.security.WebUiRequestMatcher; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class WebUiFilterTest +{ + @Test + void pagesAreUiAddressesOnly() + { + for (String page : new String[] {"/", "/query", "/admin/user", "/auth/oauth", "/dataset/adhoc/a/b"}) { + assertTrue(WebUiRequestMatcher.isPage(page), page); + } + for (String other : new String[] {"/api/v1/user", "/api", "/assets/index-1.js", "/static/images/a.png", "/favicon.ico", "/robots.txt", + "/share/report/abc", "/embed/report/abc", "/upload/a", "/h2-console", "/fonts/x"}) { + assertFalse(WebUiRequestMatcher.isPage(other), other); + } + } + + private static WebUiDocument document(String html) + { + return new WebUiDocument() + { + @Override + public Optional html() + { + return Optional.ofNullable(html); + } + }; + } + + private static MockHttpServletRequest request(String method, String path, String accept) + { + MockHttpServletRequest request = new MockHttpServletRequest(method, path); + if (accept != null) { + request.addHeader("Accept", accept); + } + return request; + } + + @Test + void browserPageRequestsGetTheUiDocument() + throws Exception + { + WebUiFilter filter = new WebUiFilter(document("MetaForge")); + MockHttpServletResponse response = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + filter.doFilter(request("GET", "/admin/user", "text/html,application/xhtml+xml"), response, chain); + assertEquals(200, response.getStatus()); + assertEquals("MetaForge", response.getContentAsString()); + assertTrue(response.getContentType().startsWith("text/html")); + assertEquals("no-cache", response.getHeader("Cache-Control")); + assertNull(chain.getRequest(), "a served page does not continue down the chain"); + } + + @Test + void apisFilesAndNonBrowserRequestsPassThrough() + throws Exception + { + WebUiFilter filter = new WebUiFilter(document("")); + for (MockHttpServletRequest request : new MockHttpServletRequest[] { + request("GET", "/api/v1/user", "text/html"), + request("GET", "/assets/app.js", "text/html"), + request("POST", "/query", "text/html"), + request("GET", "/query", "application/json"), + request("GET", "/query", null)}) { + MockFilterChain chain = new MockFilterChain(); + filter.doFilter(request, new MockHttpServletResponse(), chain); + assertNotNull(chain.getRequest(), request.getMethod() + " " + request.getRequestURI()); + } + MockFilterChain chain = new MockFilterChain(); + new WebUiFilter(document(null)).doFilter(request("GET", "/query", "text/html"), new MockHttpServletResponse(), chain); + assertNotNull(chain.getRequest(), "without a packaged UI the request continues"); + } +} diff --git a/core/metaforge-service/pom.xml b/core/metaforge-service/pom.xml index 76a21215ee..c7f853dc12 100644 --- a/core/metaforge-service/pom.xml +++ b/core/metaforge-service/pom.xml @@ -83,6 +83,22 @@ + + com.sun.mail + jakarta.mail + ${jakarta-mail.version} + + + com.sun.activation + jakarta.activation + + + + + com.sun.activation + jakarta.activation + ${jakarta-activation.version} + io.jsonwebtoken jjwt diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/activity/QueryActivity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/activity/QueryActivity.java new file mode 100644 index 0000000000..b196a74941 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/activity/QueryActivity.java @@ -0,0 +1,36 @@ +package org.devlive.metaforge.service.activity; + +import lombok.Value; + +import java.util.List; + +@Value +public class QueryActivity +{ + private int days; + private long total; + private List trend; + private List sources; + + public QueryActivity(int days, long total, List trend, List sources) + { + this.days = days; + this.total = total; + this.trend = List.copyOf(trend); + this.sources = List.copyOf(sources); + } + + @Value + public static class Day + { + private String date; + private long count; + } + + @Value + public static class Source + { + private String name; + private long count; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/annotation/SendNotification.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/annotation/SendNotification.java deleted file mode 100644 index 4bd00cb5ee..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/annotation/SendNotification.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.devlive.metaforge.service.annotation; - -import org.devlive.metaforge.service.enums.EntityType; -import org.devlive.metaforge.service.enums.NotificationType; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface SendNotification -{ - String title() default ""; - - String content() default ""; - - EntityType entityType() default EntityType.DATASET; - - NotificationType type() default NotificationType.CREATED; - - String[] channels() default {}; -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/aspect/NotificationAspect.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/aspect/NotificationAspect.java deleted file mode 100644 index f496b288ea..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/aspect/NotificationAspect.java +++ /dev/null @@ -1,257 +0,0 @@ -package org.devlive.metaforge.service.aspect; - -import com.google.common.collect.Lists; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.JoinPoint; -import org.aspectj.lang.annotation.AfterReturning; -import org.aspectj.lang.annotation.Aspect; -import org.devlive.metaforge.common.response.CommonResponse; -import org.devlive.metaforge.notify.NotifyService; -import org.devlive.metaforge.notify.model.NotifyRequest; -import org.devlive.metaforge.plugin.PluginManager; -import org.devlive.metaforge.service.annotation.SendNotification; -import org.devlive.metaforge.service.entity.BaseEntity; -import org.devlive.metaforge.service.entity.NotificationEntity; -import org.devlive.metaforge.service.entity.UserEntity; -import org.devlive.metaforge.service.enums.NotificationType; -import org.devlive.metaforge.service.itransient.NotificationTypeEntity; -import org.devlive.metaforge.service.repository.NotificationRepository; -import org.devlive.metaforge.service.repository.UserRepository; -import org.devlive.metaforge.service.security.UserDetailsService; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; - -import java.util.Objects; -import java.util.Optional; - -@Aspect -@Component -@Slf4j -@RequiredArgsConstructor -@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) -public class NotificationAspect -{ - private static final String INTERNAL_NOTIFICATION_TYPE = "Internal"; - private final NotificationRepository repository; - private final UserRepository userRepository; - private final PluginManager pluginManager; - - @AfterReturning(pointcut = "@annotation(sendNotification)", returning = "result") - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void sendNotification( - JoinPoint joinPoint, - SendNotification sendNotification, - Object result - ) - { - try { - UserEntity loginUser = UserDetailsService.getUser(); - - // 处理返回结果并发送默认通知 - processResultAndSendDefaultNotification(result, sendNotification); - - // 发送用户配置的通知 - sendCustomNotifications(loginUser, sendNotification, result); - } - catch (Exception e) { - log.error("Failed to send notification: {}", e.getMessage(), e); - } - } - - private void processResultAndSendDefaultNotification(Object result, SendNotification sendNotification) - { - if (result instanceof CommonResponse) { - @SuppressWarnings("unchecked") - CommonResponse response = (CommonResponse) result; - - // 获取当前登录用户 - UserEntity loginUser = UserDetailsService.getUser(); - - // 检查用户配置的通知类型中是否包含Internal - boolean shouldSendInternalNotification = userRepository.findByCode(loginUser.getCode()) - .map(user -> Optional.ofNullable(user.getNotifyConfigure()) - .orElse(Lists.newArrayList()) - .stream() - .anyMatch(notificationType -> { - try { - return isNotificationTypeValid(notificationType, sendNotification, INTERNAL_NOTIFICATION_TYPE, response.getData()); - } - catch (Exception e) { - log.debug("Error checking notification type: {}", e.getMessage()); - return false; - } - })) - .orElse(false); - - // 如果包含Internal类型,则发送默认的系统内部通知 - if (shouldSendInternalNotification) { - sendDefaultNotification(sendNotification, response.getData()); - } - } - } - - private boolean isNotificationTypeValid( - NotificationTypeEntity notificationType, - SendNotification sendNotification, - String type, - Object data - ) - { - if (notificationType == null || - !notificationType.isEnabled() || - notificationType.getType() == null || - !notificationType.getType().equals(type) || - notificationType.getServices() == null) { - return false; - } - - // 处理DYNAMIC类型的特殊情况 - if (sendNotification.type().equals(NotificationType.DYNAMIC)) { - // 根据data的ID判断实际操作类型 - if (data instanceof BaseEntity) { - BaseEntity entity = (BaseEntity) data; - if (entity.getId() == null) { - // 新建操作,检查services中是否包含CREATED - return notificationType.getServices().contains(NotificationType.CREATED.name()); - } - else { - // 更新操作,检查services中是否包含UPDATED - return notificationType.getServices().contains(NotificationType.UPDATED.name()); - } - } - return false; - } - else { - // 对于非DYNAMIC类型,检查具体的通知类型 - return notificationType.getServices().contains(sendNotification.type().name()); - } - } - - private void sendCustomNotifications(UserEntity loginUser, SendNotification sendNotification, Object result) - { - userRepository.findByCode(loginUser.getCode()) - .ifPresent(user -> Optional.ofNullable(user.getNotifyConfigure()) - .orElse(Lists.newArrayList()) - .stream() - .filter(Objects::nonNull) - .filter(NotificationTypeEntity::isEnabled) - .forEach(notificationType -> { - try { - if (notificationType.getServices() != null && - notificationType.getType() != null) { - // 获取结果数据 - Object data = null; - if (result instanceof CommonResponse) { - @SuppressWarnings("unchecked") - CommonResponse response = (CommonResponse) result; - data = response.getData(); - } - - // 处理DYNAMIC类型的特殊情况 - boolean servicesContainsType; - if (sendNotification.type().equals(NotificationType.DYNAMIC) && data != null) { - BaseEntity entity = (BaseEntity) data; - if (entity.getId() == null) { - // 新建操作,检查services中是否包含CREATED - servicesContainsType = notificationType.getServices().contains(NotificationType.CREATED.name()); - } - else { - // 更新操作,检查services中是否包含UPDATED - servicesContainsType = notificationType.getServices().contains(NotificationType.UPDATED.name()); - } - } - else { - // 对于非DYNAMIC类型,检查具体的通知类型 - servicesContainsType = notificationType.getServices().contains(sendNotification.type().name()); - } - - if (servicesContainsType) { - // 处理非Internal类型 - if (!INTERNAL_NOTIFICATION_TYPE.equals(notificationType.getType())) { - sendNotificationViaPlugin(notificationType.getType(), sendNotification); - } - } - } - } - catch (Exception e) { - log.debug("Error processing notification type: {}", e.getMessage()); - } - })); - } - - private void sendNotificationViaPlugin(String type, SendNotification sendNotification) - { - pluginManager.getPlugin(type) - .flatMap(plugin -> Optional.ofNullable(plugin.getService(NotifyService.class))) - .ifPresent(notifyService -> { - NotifyRequest request = createNotifyRequest(sendNotification); - notifyService.send(request); - }); - } - - private NotifyRequest createNotifyRequest(SendNotification sendNotification) - { - NotifyRequest request = new NotifyRequest(); - request.setTitle(sendNotification.title()); - request.setContent(sendNotification.content()); - return request; - } - - private void sendDefaultNotification(SendNotification sendNotification, Object configure) - { - boolean isDelete = sendNotification.type().equals(NotificationType.DELETED); - - // 处理字符串类型的配置对象 - if (configure instanceof String) { - configure = processStringConfigure((String) configure); - } - - // 构建通知实体 - NotificationEntity entity = buildNotificationEntity(sendNotification, configure, isDelete); - - // 根据动态类型设置通知类型 - if (sendNotification.type().equals(NotificationType.DYNAMIC) && configure != null) { - entity.setType(((BaseEntity) configure).getId() == null ? - NotificationType.CREATED.name() : - NotificationType.UPDATED.name()); - } - - repository.save(entity); - } - - private BaseEntity processStringConfigure(String configureCode) - { - BaseEntity baseEntity = BaseEntity.builder() - .code(configureCode) - .build(); - - // 尝试获取已通知内容是否包含当前相关信息 - repository.findFirstByEntityCode(configureCode) - .ifPresentOrElse( - entity -> baseEntity.setName(entity.getEntityName()), - () -> baseEntity.setName(configureCode) - ); - - return baseEntity; - } - - private NotificationEntity buildNotificationEntity(SendNotification sendNotification, Object configure, boolean isDelete) - { - boolean isStringConfigure = (configure instanceof String); - - return NotificationEntity.builder() - .content(sendNotification.content()) - .user(UserDetailsService.getUser()) - .type(sendNotification.type().name()) - .name(sendNotification.title()) - .entityCode(configure == null ? null : ((BaseEntity) configure).getCode()) - .entityName(configure == null ? null : ((BaseEntity) configure).getName()) - .entityType(sendNotification.entityType()) - .entityExists(!(isDelete || isStringConfigure)) - .isRead(false) - .build(); - } -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/audit/AuditPluginHandler.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/audit/AuditPluginHandler.java index efb1b44ce7..fec73ccbfc 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/audit/AuditPluginHandler.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/audit/AuditPluginHandler.java @@ -26,6 +26,7 @@ import org.devlive.metaforge.service.repository.PluginAuditRepository; import org.devlive.metaforge.service.repository.SourceRepository; import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.SystemSettingService; import org.devlive.metaforge.spi.model.Response; import org.springframework.stereotype.Component; @@ -46,6 +47,7 @@ @SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) public class AuditPluginHandler { + private final SystemSettingService systemSettings; private final ThreadLocal threadLocalPluginAudit = new ThreadLocal<>(); private final PluginAuditRepository pluginAuditRepository; @@ -53,8 +55,9 @@ public class AuditPluginHandler private final InitializerConfigure initializer; private final PluginManager pluginManager; - public AuditPluginHandler(PluginAuditRepository pluginAuditRepository, SourceRepository sourceRepository, InitializerConfigure initializer, PluginManager pluginManager) + public AuditPluginHandler(PluginAuditRepository pluginAuditRepository, SourceRepository sourceRepository, InitializerConfigure initializer, PluginManager pluginManager, SystemSettingService systemSettings) { + this.systemSettings = systemSettings; this.pluginAuditRepository = pluginAuditRepository; this.sourceRepository = sourceRepository; this.initializer = initializer; @@ -130,20 +133,20 @@ protected void handlerPlugin(final JoinPoint joinPoint, final PluginAuditEntity File tempFile = new File(requireNonNull(response.getPath())); try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) { FsRequest fsRequest = FsRequest.builder() - .access(initializer.getFsConfigure().getAccess()) - .secret(initializer.getFsConfigure().getSecret()) + .access(systemSettings.fs().getAccess()) + .secret(systemSettings.fs().getSecret()) .endpoint(workHome) - .bucket(initializer.getFsConfigure().getBucket()) + .bucket(systemSettings.fs().getBucket()) .stream(inputStream) .fileName("result") .build(); // If it is OSS third-party storage, rebuild the default directory - if (!initializer.getFsConfigure().getType().equals("LocalFs")) { - fsRequest.setEndpoint(initializer.getFsConfigure().getEndpoint()); + if (!systemSettings.fs().getType().equals("LocalFs")) { + fsRequest.setEndpoint(systemSettings.fs().getEndpoint()); fsRequest.setFileName(String.join(File.separator, user.getCode(), DateUtils.formatYMD(), String.join(File.separator, "adhoc", uniqueId), "result")); } - pluginManager.getPlugin(initializer.getFsConfigure().getType()) + pluginManager.getPlugin(systemSettings.fs().getType()) .map(v -> { FsService fsService = v.getService(FsService.class); return fsService.writer(fsRequest); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java index 9081fdc49b..f3b281ddfa 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java @@ -20,7 +20,8 @@ public class AuthBody ValidationGroup.Crud.Update.class, ValidationGroup.Crud.Auth.class }) - @Size(max = 20) + // An email when signing in by email; sign-up checks its own username rule. + @Size(max = 255) private String username; @NotBlank(groups = { @@ -32,4 +33,9 @@ public class AuthBody private String password; private Integer captcha; private Long timestamp; + private String email; + /** The email code on sign-up. */ + private String code; + /** Keep the sign-in for the remembered days when the system settings allow it. */ + private Boolean remember; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/DashboardShareBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/DashboardShareBody.java new file mode 100644 index 0000000000..2ac09cc303 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/DashboardShareBody.java @@ -0,0 +1,61 @@ +package org.devlive.metaforge.service.body; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +/** + * How a data screen is shared: with whom and how much (members and everyone), what viewers see + * (all widgets or a selection), and the public link. Sent and returned whole by the owner. + */ +@Data +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class DashboardShareBody +{ + /** VIEW, EDIT or PUBLISH for every signed-in user; null keeps the screen to its members. */ + private String everyone; + private List members = new ArrayList<>(); + /** Widget ids viewers see; null shows the whole screen. Editors always see everything. */ + private List widgets; + private PublicLink publicLink = new PublicLink(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Member + { + /** USER or ROLE */ + private String type; + private Long id; + /** Display name, filled in responses. */ + private String name; + private String permission; + } + + @Data + @NoArgsConstructor + @JsonIgnoreProperties(ignoreUnknown = true) + public static class PublicLink + { + private boolean enabled; + private boolean embed; + /** Sites allowed to embed the screen, one origin per line; empty allows any site. */ + private String origins = ""; + /** Expiry as epoch milliseconds; null never expires. */ + @JsonFormat(shape = JsonFormat.Shape.NUMBER) + private Long expiresAt; + /** Response only. */ + private String token; + /** Request only: issue a new token, invalidating the previous link. */ + private boolean rotate; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java new file mode 100644 index 0000000000..7b554c5b37 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java @@ -0,0 +1,13 @@ +package org.devlive.metaforge.service.body; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** Asks for a code mailed to an address, for sign-up (SIGNUP) or a password reset (RESET). */ +@Data +@NoArgsConstructor +public class EmailCodeBody +{ + private String email; + private String purpose; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java new file mode 100644 index 0000000000..e250868bd5 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java @@ -0,0 +1,14 @@ +package org.devlive.metaforge.service.body; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** Sets a new password for the account of an email whose code was mailed. */ +@Data +@NoArgsConstructor +public class PasswordResetBody +{ + private String email; + private String code; + private String password; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java index 2c65dab909..916a66ce70 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java @@ -187,8 +187,12 @@ public static List filterNotEmpty(List !isEmpty(v)).collect(Collectors.toList()); } + /** A field without a value is empty whatever its type, so it is not applied; a blank string is empty too. */ public static boolean isEmpty(PluginConfigureField field) { + if (field.getValue() == null) { + return true; + } boolean flag = true; switch (field.getType()) { case String: @@ -234,6 +238,9 @@ public static Map getMapValue(List configu PluginConfigureField configureField = getConfigure(configures, key); if (configureField.getType().equals(PluginConfigureFieldType.Array)) { List> list = (List>) configureField.getValue(); + if (list == null) { + return values; + } list.forEach(map -> values.put(String.valueOf(map.get(PluginConfigureFieldName.field.name())), map.get(PluginConfigureFieldName.value.name()))); } else { @@ -277,10 +284,9 @@ public static Properties convertProperties(SourceEntity entity, PluginManager pl } List fields = Lists.newArrayList(); - Optional configureExecutor = yamlConfigure.getPipelines() - .stream() - .filter(v -> v.getExecutor().equals(executor) && v.getType().equals(pipelineType)) - .findFirst(); + // 只有依赖外部引擎的执行器才声明管道配置;MetaForge 执行器直接经插件读写,没有声明时这里什么也不用做 + // Only executors driving an external engine declare pipelines; the MetaForge executor reads and writes through plugins. + Optional configureExecutor = Optional.ofNullable(yamlConfigure == null ? null : findPipelineExecutor(yamlConfigure, executor, pipelineType)); if (!configureExecutor.isPresent()) { String message = String.format("Source [ %s ] type [ %s ] is not supported pipeline type [ %s ]", entity.getId(), entity.getType(), pipelineType); Preconditions.checkArgument(!message.isEmpty(), message); @@ -359,10 +365,7 @@ public static Set convertOptions(SourceEntity entity, PluginManager plug { Set fromOptions = Sets.newHashSet(); PluginConfigure yamlConfigure = resolvePluginConfigure(pluginManager, entity.getType()); - yamlConfigure.getPipelines() - .stream() - .filter(v -> v.getExecutor().equals(executor) && v.getType().equals(pipelineType)) - .findFirst() + Optional.ofNullable(yamlConfigure == null ? null : findPipelineExecutor(yamlConfigure, executor, pipelineType)) .ifPresent(iConfigureExecutor -> iConfigureExecutor .getFields() .stream() diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/SourceColumns.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/SourceColumns.java new file mode 100644 index 0000000000..e6967a661f --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/SourceColumns.java @@ -0,0 +1,81 @@ +package org.devlive.metaforge.service.common; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.Response; +import org.devlive.metaforge.spi.model.RuntimeConfigure; + +import java.util.ArrayList; +import java.util.List; + +/** + * A table's columns as every plugin lists them through {@link PluginService#getColumns}, read by + * the standard result names (object_name, object_data_type, ...). Services that need columns use + * this instead of a source's own catalog tables, so they work for every source with column:read. + */ +public final class SourceColumns +{ + private SourceColumns() + {} + + @Getter + @AllArgsConstructor + public static final class Column + { + private final String name; + private final String type; + private final boolean nullable; + private final String defaultValue; + private final String comment; + private final boolean autoIncrement; + } + + /** The table's columns in order; empty when the table does not exist or has none. */ + public static List of(PluginService service, RuntimeConfigure configure, String database, String table) + { + Response response = service.getColumns(configure, database, table); + List columns = new ArrayList<>(); + if (response == null || !Boolean.TRUE.equals(response.getIsSuccessful()) || response.getColumns() == null) { + return columns; + } + List headers = response.getHeaders() == null ? List.of() : response.getHeaders(); + for (Object row : response.getColumns()) { + String kind = value(row, headers, "type_name"); + String name = value(row, headers, "object_name"); + if (name == null || (kind != null && !"column".equalsIgnoreCase(kind))) { + continue; + } + String type = value(row, headers, "object_data_type"); + columns.add(new Column(name, type == null ? "" : type, "YES".equalsIgnoreCase(value(row, headers, "object_nullable")), + value(row, headers, "object_default_value"), text(value(row, headers, "object_comment")), + "YES".equalsIgnoreCase(value(row, headers, "object_autoincrement")))); + } + return columns; + } + + /** One field of a row by its result name; rows are lists in header order or JSON objects. */ + private static String value(Object row, List headers, String name) + { + if (row instanceof JsonNode) { + JsonNode node = ((JsonNode) row).get(name); + if (node == null && ((JsonNode) row).isArray()) { + int index = headers.indexOf(name); + node = index < 0 ? null : ((JsonNode) row).get(index); + } + return node == null || node.isNull() ? null : node.asText(); + } + if (row instanceof List) { + int index = headers.indexOf(name); + List values = (List) row; + return index < 0 || index >= values.size() || values.get(index) == null ? null : String.valueOf(values.get(index)); + } + return null; + } + + private static String text(String value) + { + return value == null ? "" : value; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/configure/DatasetTargetSchema.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/configure/DatasetTargetSchema.java index 045f0bf231..11d7fef3a6 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/configure/DatasetTargetSchema.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/configure/DatasetTargetSchema.java @@ -37,7 +37,9 @@ public static List fields() new RuntimeConfigureField("tableDefaultEngine", RuntimeConfigureFieldType.STRING, "MergeTree", "Default table engine when materializing", false), new RuntimeConfigureField("tablePrefix", RuntimeConfigureFieldType.STRING, "metaforge_", - "Table name prefix", false) + "Table name prefix", false), + new RuntimeConfigureField("maxConcurrentSyncs", RuntimeConfigureFieldType.NUMBER, "4", + "Dataset syncs that run at the same time; further syncs wait in the queue", false) ); } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dashboard/share/DashboardViewerDocument.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dashboard/share/DashboardViewerDocument.java new file mode 100644 index 0000000000..874f7a5167 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dashboard/share/DashboardViewerDocument.java @@ -0,0 +1,157 @@ +package org.devlive.metaforge.service.dashboard.share; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * What a viewer of a shared data screen receives: only the published design, never the draft, and + * when the owner chose widgets only those widgets and the widgets inside them. Everything else, + * including the reports behind hidden widgets, stays on the server. + */ +public final class DashboardViewerDocument +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private DashboardViewerDocument() + {} + + /** + * The viewer's document: version 3 whose draft and published both hold the visible published + * design. Empty when nothing is published or the document cannot be read. + * + * @param widgets widget ids to show, or null for the whole screen + */ + public static Optional of(String configure, Set widgets) + { + Optional root = parse(configure); + if (root.isEmpty()) { + return Optional.empty(); + } + try { + if (root.get().isArray()) { + // Earlier layouts are a widget array, published once saved; ids follow the editor's legacy-. + ArrayNode visible = MAPPER.createArrayNode(); + for (int index = 0; index < root.get().size(); index++) { + if (widgets == null || widgets.contains("legacy-" + index)) { + visible.add(root.get().get(index)); + } + } + return Optional.of(MAPPER.writeValueAsString(visible)); + } + JsonNode published = root.get().path("version").asInt() == 3 ? root.get().path("published") : null; + if (published == null || !published.isObject()) { + return Optional.empty(); + } + ObjectNode design = published.deepCopy(); + if (widgets != null) { + JsonNode boards = design.path("boards"); + if (boards.isObject()) { + boards.fields().forEachRemaining(entry -> filter(entry.getValue(), widgets)); + } + } + ObjectNode document = MAPPER.createObjectNode(); + document.put("version", 3); + document.set("draft", design); + document.set("published", design.deepCopy()); + return Optional.of(MAPPER.writeValueAsString(document)); + } + catch (JsonProcessingException e) { + return Optional.empty(); + } + } + + /** Report codes the widgets of a (viewer) document draw. */ + public static Set reportCodes(String configure) + { + Set codes = new LinkedHashSet<>(); + Optional root = parse(configure); + if (root.isEmpty()) { + return codes; + } + if (root.get().isArray()) { + for (JsonNode item : root.get()) { + String code = item.path("original").path("code").asText(item.path("node").path("id").asText("")); + if (!code.isEmpty()) { + codes.add(code); + } + } + return codes; + } + for (String part : new String[] {"draft", "published"}) { + JsonNode boards = root.get().path(part).path("boards"); + if (boards.isObject()) { + boards.elements().forEachRemaining(board -> board.path("widgets").forEach(widget -> { + String code = widget.path("reportCode").asText(""); + if (!code.isEmpty()) { + codes.add(code); + } + })); + } + } + return codes; + } + + /** Widget ids of the published design, for the owner's widget picker. */ + public static Set publishedWidgetIds(String configure) + { + Set ids = new LinkedHashSet<>(); + parse(configure).map(root -> root.path("published").path("boards")).filter(JsonNode::isObject) + .ifPresent(boards -> boards.elements().forEachRemaining(board -> board.path("widgets") + .forEach(widget -> ids.add(widget.path("id").asText(""))))); + ids.remove(""); + return ids; + } + + /** Keeps the chosen widgets and everything inside them (containers hold children by parentId). */ + private static void filter(JsonNode board, Set widgets) + { + JsonNode list = board.path("widgets"); + if (!list.isArray()) { + return; + } + Map parents = new HashMap<>(); + list.forEach(widget -> parents.put(widget.path("id").asText(""), widget.path("parentId").asText(""))); + ArrayNode kept = MAPPER.createArrayNode(); + for (JsonNode widget : list) { + if (visible(widget.path("id").asText(""), widgets, parents)) { + kept.add(widget); + } + } + ((ObjectNode) board).set("widgets", kept); + } + + private static boolean visible(String id, Set widgets, Map parents) + { + Set seen = new HashSet<>(); + for (String current = id; !current.isEmpty() && seen.add(current); current = parents.getOrDefault(current, "")) { + if (widgets.contains(current)) { + return true; + } + } + return false; + } + + private static Optional parse(String configure) + { + if (configure == null || configure.isBlank()) { + return Optional.empty(); + } + try { + JsonNode root = MAPPER.readTree(configure); + return root == null || root.isNull() || root.isMissingNode() ? Optional.empty() : Optional.of(root); + } + catch (JsonProcessingException e) { + return Optional.empty(); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncActiveView.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncActiveView.java new file mode 100644 index 0000000000..cec58b2319 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncActiveView.java @@ -0,0 +1,23 @@ +package org.devlive.metaforge.service.dataset.sync; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.executor.common.RunState; + +/** + * 列表页展示的一次未结束同步:状态、进度与写入速率。progress 未知时为 -1。 + * One unfinished sync as the list page shows it; progress is -1 when unknown. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SyncActiveView +{ + private String datasetCode; + private Long historyId; + private RunState state; + private double progress; + private long rowsWritten; + private double writeRowsPerSecond; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJob.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJob.java new file mode 100644 index 0000000000..8e6d539548 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJob.java @@ -0,0 +1,134 @@ +package org.devlive.metaforge.service.dataset.sync; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; +import org.devlive.metaforge.service.entity.DatasetHistoryEntity; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +/** + * 一次正在排队或运行的数据集同步。它独占对应的 history 实体:所有修改与落库都经过本对象并在其上加锁, + * 避免状态转换、进度刷新与同步主体互相覆盖。 + * + * One queued or running dataset sync. It owns its history row: every change and save goes through + * this object under its lock, so state changes, progress updates and the sync body never overwrite + * each other. + */ +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class SyncJob +{ + private final SyncJobManager manager; + private final DatasetHistoryEntity history; + private final Long datasetId; + private final List transitions = new ArrayList<>(); + private final Consumer body; + + private volatile ExecutorMetrics metrics; + private volatile long startedAt; + private long lastPersistAt; + + SyncJob(SyncJobManager manager, DatasetHistoryEntity history, Consumer body) + { + this.manager = manager; + this.history = history; + this.body = body; + this.datasetId = history.getDatasetId(); + } + + public Long getHistoryId() + { + return history.getId(); + } + + public Long getDatasetId() + { + return datasetId; + } + + public synchronized RunState getState() + { + return history.getState(); + } + + /** 续传用的断点,没有则为 null / the checkpoint to resume from, null without one */ + public synchronized String getCheckpoint() + { + return history.getCheckpoint(); + } + + /** 首次运行前为 null,之后沿用同一任务名 / null before the first run; later runs keep the task name */ + public synchronized String getTaskName() + { + return history.getTaskName(); + } + + /** 首次运行时确定的配置 JSON,续传沿用 / the configuration JSON fixed by the first run, kept by later runs */ + public synchronized String getExecutorConfigure() + { + return history.getExecutorConfigure(); + } + + Consumer body() + { + return body; + } + + public ExecutorMetrics getMetrics() + { + return metrics; + } + + /** 进入 RUNNING 的时间戳,尚未运行为 0 / when the task started running, 0 before that */ + public long getStartedAt() + { + return startedAt; + } + + public synchronized List getTransitions() + { + return new ArrayList<>(transitions); + } + + /** 在锁内修改 history 并落库 / change the history row under the lock and save it */ + public synchronized void update(Consumer change) + { + change.accept(history); + manager.save(history); + } + + /** 状态转换,不合法时拒绝并返回 false / move to a new state, false when the move is not legal */ + public boolean transition(RunState to, String message) + { + return manager.transition(this, to, message); + } + + synchronized DatasetHistoryEntity history() + { + return history; + } + + synchronized void recordTransition(SyncTransition transition) + { + transitions.add(transition); + if (transition.getTo() == RunState.RUNNING && startedAt == 0) { + startedAt = transition.getAt(); + } + } + + void setMetrics(ExecutorMetrics metrics) + { + this.metrics = metrics; + } + + synchronized boolean persistDue(long now, long interval) + { + if (now - lastPersistAt < interval) { + return false; + } + lastPersistAt = now; + return true; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJobManager.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJobManager.java new file mode 100644 index 0000000000..796b7e9c64 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncJobManager.java @@ -0,0 +1,426 @@ +package org.devlive.metaforge.service.dataset.sync; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.common.RunStateMachine; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; +import org.devlive.metaforge.executor.configure.StageMetrics; +import org.devlive.metaforge.service.configure.DatasetTargetSchema; +import org.devlive.metaforge.service.entity.DatasetHistoryEntity; +import org.devlive.metaforge.service.repository.DatasetHistoryRepository; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.springframework.stereotype.Component; + +import javax.annotation.PreDestroy; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * 数据集同步的统一调度: + * - 全局 FIFO 队列,最多同时运行 {@code maxConcurrentSyncs} 个(系统设置 DATASET/Default,默认 4) + * - 同一数据集同一时刻只允许一个排队或运行中的同步 + * - 所有状态转换经 {@link RunStateMachine} 校验,并记录到 history.transitions + * - 指标按时间节流落库,状态变化与结束时立即落库 + * + * Schedules dataset syncs: one global FIFO queue capped at {@code maxConcurrentSyncs}, at most one + * queued or running sync per dataset, validated state transitions, and throttled metric persistence. + */ +@Slf4j +@Component +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class SyncJobManager +{ + public static final String MAX_CONCURRENT_SYNCS = "maxConcurrentSyncs"; + static final int DEFAULT_MAX_CONCURRENT_SYNCS = 4; + /** 运行中指标的落库间隔 / how often running metrics are saved */ + static final long PERSIST_INTERVAL_MS = 2000; + + private static final ObjectMapper JSON = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final TypeReference> TRANSITIONS = new TypeReference<>() {}; + + private final DatasetHistoryRepository historyRepository; + private final RuntimeConfigureService runtimeConfigureService; + private final ThreadPoolExecutor pool; + private final Map jobs = new ConcurrentHashMap<>(); + private final Map activeByDataset = new ConcurrentHashMap<>(); + + public SyncJobManager(DatasetHistoryRepository historyRepository, RuntimeConfigureService runtimeConfigureService) + { + this.historyRepository = historyRepository; + this.runtimeConfigureService = runtimeConfigureService; + AtomicInteger threads = new AtomicInteger(); + this.pool = new ThreadPoolExecutor(DEFAULT_MAX_CONCURRENT_SYNCS, DEFAULT_MAX_CONCURRENT_SYNCS, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), runnable -> { + Thread thread = new Thread(runnable, "dataset-sync-" + threads.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + this.pool.allowCoreThreadTimeOut(true); + } + + @PreDestroy + public void shutdown() + { + pool.shutdownNow(); + } + + /** + * 提交一次同步。数据集已有排队或运行中的同步时返回 empty,且不会调用 historyFactory。 + * historyFactory 负责创建并保存 CREATED 状态的 history;body 在工作线程中运行同步主体, + * 需要自行把任务推进到终态,未推进则按 FAILURE 收尾。 + * + * Submit a sync. Returns empty (without calling historyFactory) when the dataset already has a + * queued or running sync. The body must drive the job to a terminal state; otherwise it ends as FAILURE. + */ + public Optional submit(Long datasetId, Supplier historyFactory, Consumer body) + { + if (activeByDataset.putIfAbsent(datasetId, -1L) != null) { + return Optional.empty(); + } + SyncJob job; + try { + DatasetHistoryEntity history = historyFactory.get(); + job = new SyncJob(this, history, body); + jobs.put(job.getHistoryId(), job); + activeByDataset.put(datasetId, job.getHistoryId()); + job.recordTransition(new SyncTransition(null, history.getState(), System.currentTimeMillis(), null)); + } + catch (RuntimeException e) { + activeByDataset.remove(datasetId, -1L); + throw e; + } + transition(job, RunState.QUEUE, null); + resizePool(); + pool.execute(() -> run(job)); + return Optional.of(job); + } + + public Optional find(Long historyId) + { + return Optional.ofNullable(historyId == null ? null : jobs.get(historyId)); + } + + /** + * 继续一个已暂停的同步:回到队列,轮到时从断点续传。 + * Resume a paused sync: back into the queue, continuing from its checkpoint when its turn comes. + */ + public boolean resume(SyncJob job) + { + if (job.getState() != RunState.PAUSED || !transition(job, RunState.QUEUE, null)) { + return false; + } + resizePool(); + pool.execute(() -> run(job)); + return true; + } + + /** + * 让已结束的同步再次运行:同一条历史,尝试次数加一,有断点时从断点续传。数据集已有未结束的同步时返回 empty。 + * Run a finished sync again on the same history row, one attempt up, from its checkpoint if it has one. + * Returns empty while the dataset has an unfinished sync. + */ + public Optional retry(DatasetHistoryEntity history, Consumer body) + { + Long datasetId = history.getDatasetId(); + if (activeByDataset.putIfAbsent(datasetId, history.getId()) != null) { + return Optional.empty(); + } + SyncJob job = new SyncJob(this, history, body); + readTransitions(history.getTransitions()).forEach(job::recordTransition); + jobs.put(history.getId(), job); + job.update(row -> { + row.setAttempt(attemptOf(row) + 1); + row.setMessage(null); + }); + transition(job, RunState.QUEUE, null); + resizePool(); + pool.execute(() -> run(job)); + return Optional.of(job); + } + + /** + * 已结束的同步能否再次运行而不重复写入:有精确断点,或确认没有写入过任何行。异常中断时已写行数未必是最新的, + * 只认断点。 + * A finished sync may run again only with an exact checkpoint or when it certainly wrote nothing. After + * a crash the saved row count may be stale, so only a checkpoint counts. + */ + public static boolean canRetry(DatasetHistoryEntity history) + { + RunState state = history.getState(); + if (state == null || !RunStateMachine.canTransition(state, RunState.QUEUE) || !RunStateMachine.isTerminal(state)) { + return false; + } + if (history.getCheckpoint() != null) { + return true; + } + return state != RunState.INTERRUPTED && (history.getRowsWritten() == null || history.getRowsWritten() == 0); + } + + /** 所有未结束的同步,包括排队和暂停的 / every unfinished sync, queued and paused ones included */ + public List active() + { + return List.copyOf(jobs.values()); + } + + /** 该数据集是否有排队或运行中的同步 / whether the dataset has a queued or running sync */ + public boolean isActive(Long datasetId) + { + return activeByDataset.containsKey(datasetId); + } + + /** + * 取消没有在运行的同步(CREATED / QUEUE / PAUSED),直接进入 STOPPED 并释放数据集。运行中的返回 false, + * 由调用方请求执行器停止。 + * + * Cancel a sync that is not running (queued or paused); false when the executor must stop it. + */ + public boolean cancelQueued(SyncJob job) + { + synchronized (job) { + RunState state = job.getState(); + if (state != RunState.CREATED && state != RunState.QUEUE && state != RunState.PAUSED) { + return false; + } + if (!transition(job, RunState.STOPPED, state == RunState.PAUSED ? "Cancelled while paused" : "Cancelled before it started")) { + return false; + } + } + finish(job); + return true; + } + + /** 记录最新指标,按时间节流落库;返回本次是否落库 / record metrics; true when they were saved this time */ + public boolean onMetrics(SyncJob job, ExecutorMetrics metrics, boolean force) + { + job.setMetrics(metrics); + if (!force && !job.persistDue(System.currentTimeMillis(), PERSIST_INTERVAL_MS)) { + return false; + } + job.update(history -> applyMetrics(history, metrics)); + return true; + } + + /** + * 在执行器上报的指标里登记一个服务端阶段(如刷新表元数据)并立即落库。 + * Record a service-side stage (such as the table metadata refresh) in the job metrics and save it. + */ + public void stage(SyncJob job, String name, String state, long elapsedMs) + { + ExecutorMetrics metrics = job.getMetrics() == null ? new ExecutorMetrics() : job.getMetrics(); + StageMetrics stage = metrics.getStages().stream() + .filter(item -> name.equals(item.getName())) + .findFirst() + .orElseGet(() -> { + StageMetrics created = new StageMetrics(name, state, System.currentTimeMillis(), 0); + metrics.getStages().add(created); + return created; + }); + stage.setState(state); + stage.setElapsedMs(elapsedMs); + onMetrics(job, metrics, true); + } + + /** 运行中取内存视图,已结束取落库结果 / live view while running, the persisted row afterwards */ + public Optional view(Long historyId) + { + SyncJob job = jobs.get(historyId); + if (job != null) { + DatasetHistoryEntity history = job.history(); + synchronized (job) { + return Optional.of(new SyncMetricsView(historyId, history.getState(), attemptOf(history), true, + history.getMessage(), job.getMetrics(), job.getTransitions(), canRetry(history))); + } + } + return historyRepository.findById(historyId) + .map(history -> new SyncMetricsView(historyId, history.getState(), attemptOf(history), false, + history.getMessage(), readMetrics(history.getMetrics()), readTransitions(history.getTransitions()), canRetry(history))); + } + + boolean transition(SyncJob job, RunState to, String message) + { + synchronized (job) { + RunState from = job.getState(); + if (from == null || !RunStateMachine.canTransition(from, to)) { + log.warn("Sync history [ {} ] rejected state change {} -> {}", job.getHistoryId(), from, to); + return false; + } + SyncTransition transition = new SyncTransition(from, to, System.currentTimeMillis(), message); + job.recordTransition(transition); + job.update(history -> { + history.setState(to); + if (message != null) { + history.setMessage(message); + } + history.setUpdateTime(new Date(transition.getAt())); + history.setTransitions(writeJson(job.getTransitions())); + }); + log.info("Sync history [ {} ] {} -> {}", job.getHistoryId(), from, to); + return true; + } + } + + void save(DatasetHistoryEntity history) + { + try { + historyRepository.save(history); + } + catch (RuntimeException e) { + log.warn("Save sync history [ {} ] failed: {}", history.getId(), e.getMessage()); + } + } + + private void run(SyncJob job) + { + if (!transition(job, RunState.RUNNING, null)) { + // 排队期间已被取消 / cancelled while queued + return; + } + try { + job.body().accept(job); + } + catch (Throwable e) { + log.warn("Sync history [ {} ] failed", job.getHistoryId(), e); + transition(job, RunState.FAILURE, e.getMessage()); + } + finally { + // 已暂停的同步继续占着数据集,等继续或取消 / a paused sync keeps the dataset until it resumes or is cancelled + if (job.getState() != RunState.PAUSED) { + if (!RunStateMachine.isTerminal(job.getState())) { + transition(job, RunState.FAILURE, "The sync ended without a final state"); + } + finish(job); + } + } + } + + private void finish(SyncJob job) + { + ExecutorMetrics metrics = job.getMetrics(); + long startedAt = job.getStartedAt(); + job.update(history -> { + if (metrics != null) { + applyMetrics(history, metrics); + } + // 指标里的耗时不含暂停和排队 / the metrics' elapsed time excludes paused and queued time + if (metrics != null) { + history.setElapsed(metrics.getElapsedMs() / 1000); + } + else if (startedAt > 0) { + history.setElapsed((System.currentTimeMillis() - startedAt) / 1000); + } + }); + jobs.remove(job.getHistoryId(), job); + activeByDataset.remove(job.getDatasetId(), job.getHistoryId()); + } + + private void resizePool() + { + int size = maxConcurrentSyncs(); + if (size == pool.getMaximumPoolSize()) { + return; + } + if (size > pool.getMaximumPoolSize()) { + pool.setMaximumPoolSize(size); + pool.setCorePoolSize(size); + } + else { + pool.setCorePoolSize(size); + pool.setMaximumPoolSize(size); + } + log.info("Dataset sync concurrency set to {}", size); + } + + int maxConcurrentSyncs() + { + try { + String raw = runtimeConfigureService.getEffective(RuntimeConfigureService.CATEGORY_DATASET, DatasetTargetSchema.NAME, DatasetTargetSchema.fields()) + .get(MAX_CONCURRENT_SYNCS); + int value = raw == null ? DEFAULT_MAX_CONCURRENT_SYNCS : Integer.parseInt(raw.trim()); + return value > 0 ? value : DEFAULT_MAX_CONCURRENT_SYNCS; + } + catch (RuntimeException e) { + return DEFAULT_MAX_CONCURRENT_SYNCS; + } + } + + private static void applyMetrics(DatasetHistoryEntity history, ExecutorMetrics metrics) + { + history.setRowsRead(metrics.getRowsRead()); + history.setRowsWritten(metrics.getRowsWritten()); + history.setRowsSkipped(metrics.getRowsSkipped()); + history.setRowsFailed(metrics.getRowsFailed()); + history.setBytesRead(metrics.getBytesRead()); + history.setBytesWritten(metrics.getBytesWritten()); + history.setProcessedCount(metrics.getRowsWritten()); + if (metrics.getTotalRows() >= 0) { + history.setTotalCount(metrics.getTotalRows()); + } + if (metrics.getProgress() >= 0) { + history.setProgress(BigDecimal.valueOf(metrics.getProgress()).setScale(2, RoundingMode.HALF_UP)); + } + history.setMetrics(writeJson(metrics)); + } + + private static int attemptOf(DatasetHistoryEntity history) + { + return history.getAttempt() == null ? 1 : history.getAttempt(); + } + + private static String writeJson(Object value) + { + try { + return JSON.writeValueAsString(value); + } + catch (JsonProcessingException e) { + log.warn("Serialize sync state failed: {}", e.getMessage()); + return null; + } + } + + static ExecutorMetrics readMetrics(String raw) + { + if (raw == null || raw.isBlank()) { + return null; + } + try { + return JSON.readValue(raw, ExecutorMetrics.class); + } + catch (JsonProcessingException e) { + log.warn("Read sync metrics failed: {}", e.getMessage()); + return null; + } + } + + static List readTransitions(String raw) + { + if (raw == null || raw.isBlank()) { + return Collections.emptyList(); + } + try { + return JSON.readValue(raw, TRANSITIONS); + } + catch (JsonProcessingException e) { + log.warn("Read sync transitions failed: {}", e.getMessage()); + return Collections.emptyList(); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncMetricsView.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncMetricsView.java new file mode 100644 index 0000000000..dd883554ce --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncMetricsView.java @@ -0,0 +1,33 @@ +package org.devlive.metaforge.service.dataset.sync; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; + +import java.util.List; + +/** + * 一次同步的实时视图:状态、尝试次数、最新指标与状态转换记录。live 表示数据来自内存中的运行任务, + * 否则来自落库的最终结果。 + * + * Live view of one sync run; {@code live} tells a running task from a persisted, finished one. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class SyncMetricsView +{ + private Long historyId; + private RunState state; + private int attempt; + private boolean live; + private String message; + private ExecutorMetrics metrics; + private List transitions; + /** 已结束的同步能否继续运行而不重复写入 / whether a finished sync can run again without writing anything twice */ + private boolean retryable; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncTransition.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncTransition.java new file mode 100644 index 0000000000..b071ded2b1 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/dataset/sync/SyncTransition.java @@ -0,0 +1,35 @@ +package org.devlive.metaforge.service.dataset.sync; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.executor.common.RunState; + +import java.util.Date; + +/** + * 一次状态转换记录:从哪个状态到哪个状态、发生时间(毫秒时间戳)与原因。 + * One recorded state change of a sync task. + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +@NoArgsConstructor +@AllArgsConstructor +public class SyncTransition +{ + private RunState from; + private RunState to; + private long at; + private String message; + + /** The transition time as the server formats every date; {@link #at} stays for ordering. */ + @JsonProperty(access = JsonProperty.Access.READ_ONLY) + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss") + public Date getTime() + { + return at > 0 ? new Date(at) : null; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ConfigureEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ConfigureEntity.java index 60696ea152..8f70135cb6 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ConfigureEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ConfigureEntity.java @@ -17,7 +17,7 @@ * * Generic configuration row keyed by (category, name). * - category: e.g. "EXECUTOR", "DATASET" - * - name: e.g. "Local", "Seatunnel", or a dataset target identifier + * - name: e.g. "MetaForge", "Seatunnel", or a dataset target identifier * - configure: JSON-serialized map of effective field values */ @Data diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DashboardEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DashboardEntity.java index 5b479ccb28..f4362a1219 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DashboardEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DashboardEntity.java @@ -58,4 +58,14 @@ public class DashboardEntity @Transient @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private Set reports; + + /** The signed-in user's permission on this screen (OWNER, VIEW, EDIT or PUBLISH); filled on reads. */ + @Transient + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String permission; + + /** Who owns a screen shared with the signed-in user; filled for shared screens. */ + @Transient + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String ownerName; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DatasetHistoryEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DatasetHistoryEntity.java index aebc99d987..f8cff86727 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DatasetHistoryEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/DatasetHistoryEntity.java @@ -1,5 +1,6 @@ package org.devlive.metaforge.service.entity; +import com.fasterxml.jackson.annotation.JsonIgnore; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.AllArgsConstructor; import lombok.Data; @@ -46,6 +47,42 @@ public class DatasetHistoryEntity @Column("progress") private BigDecimal progress; + @Column("rows_read") + private Long rowsRead; + + @Column("rows_written") + private Long rowsWritten; + + @Column("rows_skipped") + private Long rowsSkipped; + + @Column("rows_failed") + private Long rowsFailed; + + @Column("bytes_read") + private Long bytesRead; + + @Column("bytes_written") + private Long bytesWritten; + + /** 第几次尝试,从 1 开始 / attempt number, starting at 1 */ + @Column("attempt") + private Integer attempt; + + /** 最近一次(结束时为最终)指标快照 JSON / latest (final once finished) metrics snapshot as JSON */ + @Column("metrics") + private String metrics; + + /** 状态转换记录 JSON 数组 / state transitions as a JSON array */ + @Column("transitions") + private String transitions; + + /** 未完成时用于精确续传的断点(内容由执行器定义)/ checkpoint to resume an unfinished run exactly, owned by the executor */ + @JsonIgnore + // checkpoint itself is a T-SQL reserved word and breaks SQL Server reads. + @Column("checkpoint_payload") + private String checkpoint; + @Column("task_name") private String taskName; diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java index ab717e1824..a65881c4e3 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java @@ -1,8 +1,10 @@ package org.devlive.metaforge.service.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonView; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -15,6 +17,8 @@ import org.springframework.data.relational.core.mapping.Column; import org.springframework.data.relational.core.mapping.Table; +import java.util.Date; + @Data @SuperBuilder @ToString @@ -68,6 +72,50 @@ public class ReportEntity @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private Long datasetId; + // Public link and iframe embedding. The token is the only public identifier of a shared report; + // only its owner reads these fields and they change through the share endpoint, never a save. + @Column("share_token") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String shareToken; + + @Column("share_enabled") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareEnabled; + + @Column("share_embed") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareEmbed; + + // Origins allowed to frame the report, one per line; empty allows any site. + @Column("share_origins") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String shareOrigins; + + // The link stops working after this instant; null never expires. Sent as epoch milliseconds. + @Column("share_expires_at") + @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private Date shareExpiresAt; + + @Column("share_show_title") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + @Builder.Default + private boolean shareShowTitle = true; + + @Column("share_show_description") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + @Builder.Default + private boolean shareShowDescription = true; + + // Offer the data as a table and as a CSV download on the public page + @Column("share_allow_data") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareAllowData; + + @Column("share_allow_download") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareAllowDownload; + @Transient @Reference("datasetId") @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java index cea59972f0..f850b82cbb 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java @@ -137,10 +137,6 @@ public class SourceEntity @JsonProperty(value = "configure") private PluginConfigure configure; - @Transient - @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) - private Boolean isSupportMeta; - public void setOriginalConfigure(String configure) { this.originalConfigure = configure; diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java index 5dbc1c7341..78115003b1 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java @@ -66,6 +66,10 @@ public class UserEntity @JsonView(value = {EntityView.NoneView.class}) private String password; + @Column("email") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String email; + @Column("chat_configure") @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private String chatConfigure; @@ -78,6 +82,11 @@ public class UserEntity @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private UserEditorEntity editorConfigure; + // Code of the user's own published data screen opened as their home page; null for the default home. + @Column("home_page") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String homePage; + @Column("avatar_configure") @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private AvatarEntity avatarConfigure; @@ -108,7 +117,8 @@ public class UserEntity @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private Long unreadCount; - @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + // Holds channel secrets: read through /api/v1/user/notify, which masks them, never with the user. + @JsonIgnore public List getNotifyConfigure() { if (StringUtils.isEmpty(notifyConfigureValue)) { diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserOAuthEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserOAuthEntity.java new file mode 100644 index 0000000000..ae6858539c --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserOAuthEntity.java @@ -0,0 +1,38 @@ +package org.devlive.metaforge.service.entity; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import java.util.Date; + +/** A user's account at a third-party sign-in provider, found again by the provider's subject id. */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Table("metaforge_user_oauth") +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class UserOAuthEntity +{ + @Id + private Long id; + + @Column("user_id") + private Long userId; + + // github, google or oidc + @Column("provider") + private String provider; + + @Column("subject") + private String subject; + + @Column("create_time") + private Date createTime; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/EntityType.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/EntityType.java index e90f5c924f..3ede6eca79 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/EntityType.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/EntityType.java @@ -1,6 +1,13 @@ package org.devlive.metaforge.service.enums; +/** What a notification links to on the site. */ public enum EntityType { - DATASET + DATASET, + SOURCE, + REPORT, + DASHBOARD, + WORKFLOW, + USER, + PLUGIN } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationScene.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationScene.java new file mode 100644 index 0000000000..cefb2b18c2 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationScene.java @@ -0,0 +1,145 @@ +package org.devlive.metaforge.service.enums; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Everything a user can be notified about, grouped by category. A user subscribes each enabled + * channel to scenes; the web UI shows them as notify.category.* and notify.scene.*. Scenes marked + * admin concern system administration and are offered to administrators only. + */ +public enum NotificationScene +{ + DATASET_CREATED(Category.DATASET), + DATASET_UPDATED(Category.DATASET), + DATASET_DELETED(Category.DATASET), + DATASET_SYNC_SUCCEEDED(Category.DATASET), + DATASET_SYNC_FAILED(Category.DATASET), + + SOURCE_CREATED(Category.SOURCE), + SOURCE_UPDATED(Category.SOURCE), + SOURCE_DELETED(Category.SOURCE), + + REPORT_CREATED(Category.REPORT), + REPORT_DELETED(Category.REPORT), + REPORT_SHARED(Category.REPORT), + REPORT_UNSHARED(Category.REPORT), + DASHBOARD_CREATED(Category.REPORT), + + IMPORT_SUCCEEDED(Category.TRANSFER), + IMPORT_FAILED(Category.TRANSFER), + EXPORT_SUCCEEDED(Category.TRANSFER), + EXPORT_FAILED(Category.TRANSFER), + + PASSWORD_CHANGED(Category.ACCOUNT), + OAUTH_BOUND(Category.ACCOUNT), + + PLUGIN_INSTALLED(Category.PLUGIN, true), + PLUGIN_UNINSTALLED(Category.PLUGIN, true), + PLUGIN_STATE_CHANGED(Category.PLUGIN, true); + + public enum Category + { + DATASET, SOURCE, REPORT, TRANSFER, ACCOUNT, PLUGIN + } + + // Subscriptions saved before scenes existed named only an operation, which always meant datasets. + private static final Map> LEGACY = Map.of( + "CREATED", List.of(DATASET_CREATED), + "UPDATED", List.of(DATASET_UPDATED), + "DELETED", List.of(DATASET_DELETED), + "DYNAMIC", List.of(DATASET_CREATED, DATASET_UPDATED), + "SYNCDATA", List.of(DATASET_SYNC_SUCCEEDED, DATASET_SYNC_FAILED)); + + private final Category category; + private final boolean admin; + + NotificationScene(Category category) + { + this(category, false); + } + + NotificationScene(Category category, boolean admin) + { + this.category = category; + this.admin = admin; + } + + public Category getCategory() + { + return category; + } + + public boolean isAdmin() + { + return admin; + } + + /** Notifications that belong to no scene, such as channel tests, are system notices. */ + public static final String SYSTEM = "SYSTEM"; + + /** The category a stored notification type belongs to: its scene's, dataset for old operation names, else system. */ + public static String categoryOf(String type) + { + if (type != null && LEGACY.containsKey(type)) { + return Category.DATASET.name(); + } + return of(type).map(scene -> scene.getCategory().name()).orElse(SYSTEM); + } + + /** Every stored type that falls under a scene category, old operation names included. */ + public static Set typesOf(Category category) + { + Set types = new LinkedHashSet<>(); + for (NotificationScene scene : values()) { + if (scene.getCategory() == category) { + types.add(scene.name()); + } + } + if (category == Category.DATASET) { + types.addAll(LEGACY.keySet()); + } + return types; + } + + /** Every type that belongs to some scene category; anything else is a system notice. */ + public static Set knownTypes() + { + Set types = new LinkedHashSet<>(LEGACY.keySet()); + for (NotificationScene scene : values()) { + types.add(scene.name()); + } + return types; + } + + public static Optional of(String name) + { + return Arrays.stream(values()).filter(scene -> scene.name().equals(name)).findFirst(); + } + + /** Scene names for saved subscriptions, upgrading old operation names; unknown names are dropped. */ + public static Set normalize(Iterable names) + { + Set scenes = new LinkedHashSet<>(); + if (names == null) { + return scenes; + } + for (String name : names) { + if (name == null) { + continue; + } + List legacy = LEGACY.get(name); + if (legacy != null) { + legacy.forEach(scene -> scenes.add(scene.name())); + } + else { + of(name).ifPresent(scene -> scenes.add(scene.name())); + } + } + return scenes; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationType.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationType.java deleted file mode 100644 index f909a3c399..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/NotificationType.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.devlive.metaforge.service.enums; - -public enum NotificationType -{ - CREATED, - UPDATED, - DELETED, - DYNAMIC, - SYNCDATA -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/SharePermission.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/SharePermission.java new file mode 100644 index 0000000000..8e09b642fc --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/enums/SharePermission.java @@ -0,0 +1,35 @@ +package org.devlive.metaforge.service.enums; + +import java.util.Optional; + +/** + * What a user may do with a data screen shared with them, from least to most. The owner is above all + * of these: only the owner deletes a screen or changes how it is shared. + */ +public enum SharePermission +{ + /** See the published screen, limited to the widgets the owner chose. */ + VIEW, + /** Also open the designer and save drafts; the published screen stays as it was. */ + EDIT, + /** Also publish drafts. */ + PUBLISH, + /** The owner. Never granted; resolved for the screen's own user. */ + OWNER; + + public boolean atLeast(SharePermission other) + { + return ordinal() >= other.ordinal(); + } + + /** A grantable permission by name, ignoring case; OWNER is never grantable. */ + public static Optional grantable(String value) + { + for (SharePermission permission : values()) { + if (permission != OWNER && permission.name().equalsIgnoreCase(value == null ? "" : value.trim())) { + return Optional.of(permission); + } + } + return Optional.empty(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEvent.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEvent.java index 791ddfbe7f..574702051f 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEvent.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEvent.java @@ -1,40 +1,29 @@ package org.devlive.metaforge.service.event; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Builder; import lombok.Getter; -import org.devlive.metaforge.service.entity.BaseEntity; import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.enums.EntityType; -import org.devlive.metaforge.service.enums.NotificationType; +import org.devlive.metaforge.service.enums.NotificationScene; +/** + * Something a user may be notified about: the scene, the user it concerns, a readable title and + * content, and the entity it links to on the site. + */ @Getter -@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +@Builder +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Immutable event carrying entity references") public class NotificationEvent { - private final BaseEntity data; + private final NotificationScene scene; private final UserEntity user; private final String title; private final String content; private final EntityType entityType; - private final NotificationType type; - private final String[] channels; - - public NotificationEvent( - BaseEntity data, - UserEntity user, - String title, - String content, - EntityType entityType, - NotificationType type, - String[] channels - ) - { - this.data = data; - this.user = user; - this.title = title; - this.content = content; - this.entityType = entityType; - this.type = type; - this.channels = channels; - } + private final String entityCode; + private final String entityName; + /** Whether the entity still exists, so the site can link to it. */ + @Builder.Default + private final boolean entityExists = true; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventListener.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventListener.java index 12d4e34555..37701147c1 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventListener.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventListener.java @@ -1,236 +1,30 @@ package org.devlive.metaforge.service.event; -import com.google.common.collect.Lists; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.devlive.metaforge.notify.NotifyService; -import org.devlive.metaforge.notify.model.NotifyRequest; -import org.devlive.metaforge.plugin.PluginManager; -import org.devlive.metaforge.service.entity.BaseEntity; -import org.devlive.metaforge.service.entity.NotificationEntity; -import org.devlive.metaforge.service.entity.UserEntity; -import org.devlive.metaforge.service.enums.NotificationType; -import org.devlive.metaforge.service.itransient.NotificationTypeEntity; -import org.devlive.metaforge.service.repository.NotificationRepository; -import org.devlive.metaforge.service.repository.UserRepository; -import org.springframework.context.event.EventListener; +import org.devlive.metaforge.service.notify.NotificationDispatcher; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; -import java.util.Objects; -import java.util.Optional; - -@Component @Slf4j +@Component @RequiredArgsConstructor -@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "The dispatcher is a Spring-managed singleton") public class NotificationEventListener { - private static final String INTERNAL_NOTIFICATION_TYPE = "Internal"; - private final NotificationRepository repository; - private final UserRepository userRepository; - private final PluginManager pluginManager; + private final NotificationDispatcher dispatcher; - @EventListener - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void sendNotification(NotificationEvent event) + // Runs after commit when raised inside a transaction, right away otherwise. + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true) + public void onNotification(NotificationEvent event) { try { - UserEntity loginUser = event.getUser(); - - // 处理返回结果并发送默认通知 - processResultAndSendDefaultNotification(event); - - // 发送用户配置的通知 - sendCustomNotifications(loginUser, event); - } - catch (Exception e) { - log.error("Failed to send notification via event: {}", e.getMessage(), e); - } - } - - private void processResultAndSendDefaultNotification(NotificationEvent event) - { - // 获取当前登录用户 - UserEntity loginUser = event.getUser(); - - // 检查用户配置的通知类型中是否包含Internal - boolean shouldSendInternalNotification = userRepository.findByCode(loginUser.getCode()) - .map(user -> Optional.ofNullable(user.getNotifyConfigure()) - .orElse(Lists.newArrayList()) - .stream() - .anyMatch(notificationType -> { - try { - return isNotificationTypeValid(notificationType, event.getType(), INTERNAL_NOTIFICATION_TYPE, event.getData()); - } - catch (Exception e) { - log.debug("Error checking notification type: {}", e.getMessage()); - return false; - } - })) - .orElse(false); - - // 如果包含Internal类型,则发送默认的系统内部通知 - if (shouldSendInternalNotification) { - sendDefaultNotification(event, event.getData()); + dispatcher.dispatch(event); } - } - - private boolean isNotificationTypeValid( - NotificationTypeEntity notificationType, - NotificationType eventType, - String type, - Object data - ) - { - if (notificationType == null || - !notificationType.isEnabled() || - notificationType.getType() == null || - !notificationType.getType().equals(type) || - notificationType.getServices() == null) { - return false; - } - - // 处理DYNAMIC类型的特殊情况 - if (eventType.equals(NotificationType.DYNAMIC)) { - // 根据data的ID判断实际操作类型 - if (data instanceof BaseEntity) { - BaseEntity entity = (BaseEntity) data; - if (entity.getId() == null) { - // 新建操作,检查services中是否包含CREATED - return notificationType.getServices().contains(NotificationType.CREATED.name()); - } - else { - // 更新操作,检查services中是否包含UPDATED - return notificationType.getServices().contains(NotificationType.UPDATED.name()); - } - } - return false; - } - else { - // 对于非DYNAMIC类型,检查具体的通知类型 - return notificationType.getServices().contains(eventType.name()); + catch (RuntimeException e) { + log.warn("Notification {} failed: {}", event.getScene(), e.getClass().getSimpleName()); } } - - private void sendCustomNotifications(UserEntity loginUser, NotificationEvent event) - { - userRepository.findByCode(loginUser.getCode()) - .ifPresent(user -> Optional.ofNullable(user.getNotifyConfigure()) - .orElse(Lists.newArrayList()) - .stream() - .filter(Objects::nonNull) - .filter(NotificationTypeEntity::isEnabled) - .forEach(notificationType -> { - try { - if (notificationType.getServices() != null && notificationType.getType() != null) { - // 获取结果数据 - Object data = event.getData(); - - // 处理DYNAMIC类型的特殊情况 - boolean servicesContainsType; - if (event.getType().equals(NotificationType.DYNAMIC) && data != null) { - BaseEntity entity = (BaseEntity) data; - if (entity.getId() == null) { - // 新建操作,检查services中是否包含CREATED - servicesContainsType = notificationType.getServices().contains(NotificationType.CREATED.name()); - } - else { - // 更新操作,检查services中是否包含UPDATED - servicesContainsType = notificationType.getServices().contains(NotificationType.UPDATED.name()); - } - } - else { - // 对于非DYNAMIC类型,检查具体的通知类型 - servicesContainsType = notificationType.getServices().contains(event.getType().name()); - } - - if (servicesContainsType) { - // 处理非Internal类型 - if (!INTERNAL_NOTIFICATION_TYPE.equals(notificationType.getType())) { - sendNotificationViaPlugin(notificationType.getType(), event); - } - } - } - } - catch (Exception e) { - log.debug("Error processing notification type: {}", e.getMessage()); - } - })); - } - - private void sendNotificationViaPlugin(String type, NotificationEvent event) - { - pluginManager.getPlugin(type) - .flatMap(plugin -> Optional.ofNullable(plugin.getService(NotifyService.class))) - .ifPresent(notifyService -> { - NotifyRequest request = createNotifyRequest(event); - notifyService.send(request); - }); - } - - private NotifyRequest createNotifyRequest(NotificationEvent event) - { - NotifyRequest request = new NotifyRequest(); - request.setTitle(event.getTitle()); - request.setContent(event.getContent()); - return request; - } - - private void sendDefaultNotification(NotificationEvent event, Object configure) - { - boolean isDelete = event.getType().equals(NotificationType.DELETED); - - // 处理字符串类型的配置对象 - if (configure instanceof String) { - configure = processStringConfigure((String) configure); - } - - // 构建通知实体 - NotificationEntity entity = buildNotificationEntity(event, configure, isDelete); - - // 根据动态类型设置通知类型 - if (event.getType().equals(NotificationType.DYNAMIC) && configure != null) { - entity.setType(((BaseEntity) configure).getId() == null ? - NotificationType.CREATED.name() : - NotificationType.UPDATED.name()); - } - - repository.save(entity); - } - - private BaseEntity processStringConfigure(String configureCode) - { - BaseEntity baseEntity = BaseEntity.builder() - .code(configureCode) - .build(); - - // 尝试获取已通知内容是否包含当前相关信息 - repository.findFirstByEntityCode(configureCode) - .ifPresentOrElse( - entity -> baseEntity.setName(entity.getEntityName()), - () -> baseEntity.setName(configureCode) - ); - - return baseEntity; - } - - private NotificationEntity buildNotificationEntity(NotificationEvent event, Object configure, boolean isDelete) - { - boolean isStringConfigure = (configure instanceof String); - - return NotificationEntity.builder() - .content(event.getContent()) - .user(event.getUser()) - .type(event.getType().name()) - .name(event.getTitle()) - .entityCode(configure == null ? null : ((BaseEntity) configure).getCode()) - .entityName(configure == null ? null : ((BaseEntity) configure).getName()) - .entityType(event.getEntityType()) - .entityExists(!(isDelete || isStringConfigure)) - .isRead(false) - .build(); - } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventPublisher.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventPublisher.java index 19ae6099b5..6bae94a8b9 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventPublisher.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/event/NotificationEventPublisher.java @@ -1,52 +1,74 @@ package org.devlive.metaforge.service.event; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.RequiredArgsConstructor; -import org.devlive.metaforge.service.entity.BaseEntity; import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.enums.EntityType; -import org.devlive.metaforge.service.enums.NotificationType; +import org.devlive.metaforge.service.enums.NotificationScene; import org.devlive.metaforge.service.security.UserDetailsService; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; +/** + * Raises notifications. Delivery happens after the surrounding transaction commits, so a change + * that rolls back notifies nobody, and it never fails the caller. + */ @Service @RequiredArgsConstructor +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "The event publisher is a Spring-managed singleton") public class NotificationEventPublisher { + /** Publishes nothing; stands in where no Spring context wires the real one, as in unit tests. */ + public static final NotificationEventPublisher NONE = new NotificationEventPublisher(event -> { + }); + private final ApplicationEventPublisher eventPublisher; - /** - * 发布通知事件 - * - * @param data 相关数据实体 - * @param title 通知标题 - * @param content 通知内容 - * @param entityType 实体类型 - * @param type 通知类型 - * @param channels 通知渠道 - */ - public void publishNotificationEvent( - BaseEntity data, - String title, - String content, - EntityType entityType, - NotificationType type, - String[] channels - ) + public void publish(NotificationEvent event) + { + if (event != null && event.getUser() != null && event.getScene() != null) { + eventPublisher.publishEvent(event); + } + } + + /** Notifies the signed-in user about an entity. */ + public void publish(NotificationScene scene, String title, String content, EntityType entityType, String entityCode, String entityName) + { + publish(scene, currentUser(), title, content, entityType, entityCode, entityName); + } + + public void publish(NotificationScene scene, UserEntity user, String title, String content, EntityType entityType, String entityCode, String entityName) + { + publish(NotificationEvent.builder() + .scene(scene) + .user(user) + .title(title) + .content(content) + .entityType(entityType) + .entityCode(entityCode) + .entityName(entityName) + .entityExists(!scene.name().endsWith("_DELETED")) + .build()); + } + + /** Notifies the user with the given code, for work that finishes off the request thread. */ + public void publishTo(String userCode, NotificationScene scene, String title, String content, EntityType entityType, String entityCode, String entityName) + { + if (userCode == null) { + return; + } + UserEntity user = new UserEntity(); + user.setCode(userCode); + publish(scene, user, title, content, entityType, entityCode, entityName); + } + + private static UserEntity currentUser() { - // 获取当前用户 - UserEntity user = UserDetailsService.getUser(); - - // 发布事件 - NotificationEvent event = new NotificationEvent( - data, - user, - title, - content, - entityType, - type, - channels - ); - eventPublisher.publishEvent(event); + try { + return UserDetailsService.getUser(); + } + catch (RuntimeException e) { + return null; + } } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java new file mode 100644 index 0000000000..6e7eda717f --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java @@ -0,0 +1,69 @@ +package org.devlive.metaforge.service.exporter; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** What a database export writes: which tables, their fields and rows, the format and the output. */ +@Data +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class ExportPlan +{ + private List tables = new ArrayList<>(); + /** all: structure and data; data: data only. */ + private String content = "all"; + private Structure structure = new Structure(); + /** An installed convert plugin, e.g. JsonConvert. */ + private String format; + /** Values of the plugin's declared options. */ + private Map options = new LinkedHashMap<>(); + private String fileName; + /** zip: a file per table packed into one ZIP; merge: one file for every table (only for plugins that merge). */ + private String organize = "zip"; + private boolean compress = true; + private int batchSize = 1000; + private int concurrency = 1; + /** batch: read each table in pages of batchSize; single: one query per table. */ + private String reading = "batch"; + /** stop: end the job at the first failing table; skip: export the others. */ + private String failure = "stop"; + /** Keep the job and its file for two hours after the download. */ + private boolean keepRecord = true; + + @Data + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) + public static class TablePlan + { + private String name; + /** Fields to export; empty exports every field. */ + private List fields = new ArrayList<>(); + /** Conditions the rows must all meet; empty exports every row. */ + private List filters = new ArrayList<>(); + } + + @Data + public static class Filter + { + private String column; + /** =, !=, >, >=, <, <=, LIKE, NOT LIKE, IS NULL, IS NOT NULL */ + private String operator = "="; + private String value; + } + + @Data + public static class Structure + { + private boolean create = true; + private boolean indexes = true; + private boolean drop; + } + + public boolean includesStructure() + { + return "all".equals(content) && (structure.isCreate() || structure.isDrop()); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java new file mode 100644 index 0000000000..31bd01a147 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java @@ -0,0 +1,115 @@ +package org.devlive.metaforge.service.exporter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** The SQL a database export runs: row selects with fields, filters and pages, and the schema file. */ +public final class ExportStatements +{ + public static final Set OPERATORS = Set.of("=", "!=", ">", ">=", "<", "<=", "LIKE", "NOT LIKE", "IS NULL", "IS NOT NULL"); + private static final Pattern INDEX_LINE = Pattern.compile("^\\s*(UNIQUE |FULLTEXT |SPATIAL )?(KEY|INDEX)\\s|^\\s*CONSTRAINT\\s", Pattern.CASE_INSENSITIVE); + + private final UnaryOperator identifier; + private final UnaryOperator literal; + + public ExportStatements(UnaryOperator identifier, UnaryOperator literal) + { + this.identifier = identifier; + this.literal = literal; + } + + public String target(String database, String table) + { + return identifier.apply(database) + "." + identifier.apply(table); + } + + /** WHERE clause of a table's filters, empty without filters. */ + public String where(List filters) + { + if (filters == null || filters.isEmpty()) { + return ""; + } + List conditions = new ArrayList<>(); + for (ExportPlan.Filter filter : filters) { + String operator = filter.getOperator() == null ? "=" : filter.getOperator().toUpperCase(Locale.ROOT); + if (!OPERATORS.contains(operator)) { + throw new IllegalArgumentException("Unsupported operator " + filter.getOperator()); + } + String column = identifier.apply(filter.getColumn()); + conditions.add(operator.startsWith("IS") ? column + " " + operator : column + " " + operator + " " + literal.apply(filter.getValue() == null ? "" : filter.getValue())); + } + return " WHERE " + String.join(" AND ", conditions); + } + + /** The select of one page (limit > 0) or of every row (limit <= 0). */ + public String select(String database, ExportPlan.TablePlan table, long limit, long offset) + { + String fields = table.getFields() == null || table.getFields().isEmpty() ? "*" + : table.getFields().stream().map(identifier).collect(Collectors.joining(", ")); + String sql = "SELECT " + fields + " FROM " + target(database, table.getName()) + where(table.getFilters()); + return limit > 0 ? sql + " LIMIT " + limit + " OFFSET " + offset : sql; + } + + public String count(String database, ExportPlan.TablePlan table) + { + return "SELECT COUNT(*) FROM " + target(database, table.getName()) + where(table.getFilters()); + } + + /** The schema.sql part of one table: DROP and CREATE as chosen, secondary indexes and constraints left out on request. */ + public String schema(String database, String table, String create, ExportPlan.Structure structure) + { + StringBuilder sql = new StringBuilder(); + if (structure.isDrop()) { + sql.append("DROP TABLE IF EXISTS ").append(target(database, table)).append(";\n"); + } + if (structure.isCreate() && create != null && !create.isEmpty()) { + String statement = structure.isIndexes() ? create : withoutIndexes(create); + sql.append(statement.trim().replaceAll(";+$", "")).append(";\n"); + } + return sql.toString(); + } + + /** CREATE TABLE without KEY, INDEX and CONSTRAINT lines; columns and the primary key stay. */ + static String withoutIndexes(String create) + { + List lines = new ArrayList<>(); + for (String line : create.split("\n")) { + if (!INDEX_LINE.matcher(line).find()) { + lines.add(line); + } + } + // The line before the closing parenthesis must not end with a comma any more. + for (int i = lines.size() - 1; i > 0; i--) { + if (lines.get(i).trim().startsWith(")")) { + String previous = lines.get(i - 1); + lines.set(i - 1, previous.replaceAll(",\\s*$", "")); + break; + } + } + return String.join("\n", lines); + } + + /** The file of a table, with the plugin's extension. */ + public static String fileName(String table, String extension) + { + return table + "." + extension; + } + + /** A name for a file: path separators and reserved characters replaced. */ + public static String safeName(String name, String fallback) + { + String safe = name == null ? "" : name.trim().replaceAll("[\\\\/:*?\"<>|\\s]+", "_"); + return safe.isEmpty() ? fallback : safe; + } + + public static boolean known(Collection names, String name) + { + return name != null && names.contains(name); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java index 5c1f5d2ec4..a92a8c62dd 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java @@ -103,7 +103,8 @@ public void init() cacheExpiration = 2L; } log.info("Metaforge cache expiration: {}", this.cacheExpiration); - if (ObjectUtils.isEmpty(cache) && captchaEnable) { + // The captcha can be turned on in system settings at any time, so its cache always exists. + if (ObjectUtils.isEmpty(cache)) { cache = CacheBuilder.newBuilder() .expireAfterWrite(cacheExpiration, TimeUnit.MINUTES) .maximumSize(cacheMaximum) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/NotificationTypeEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/NotificationTypeEntity.java index 35e752ab6a..4919608e28 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/NotificationTypeEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/NotificationTypeEntity.java @@ -8,6 +8,7 @@ import lombok.NoArgsConstructor; import org.devlive.metaforge.common.view.EntityView; +import java.util.Map; import java.util.Set; @Data @@ -25,4 +26,8 @@ public class NotificationTypeEntity @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private Set services; + + // The user's settings for this channel, by field name; secret values never leave the server. + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private Map configure; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/CategoryConfiguration.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/CategoryConfiguration.java deleted file mode 100644 index 597040301c..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/CategoryConfiguration.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.devlive.metaforge.service.itransient.configuration; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonView; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; -import org.devlive.metaforge.common.view.EntityView; - -import java.util.Set; - -@Data -@ToString -@NoArgsConstructor -@AllArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) -@JsonInclude(JsonInclude.Include.NON_NULL) -@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2", "EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC"}) -public class CategoryConfiguration -{ - private String value; - private String label; - private Set supportExecutors; -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/Configuration.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/Configuration.java index 53cd928644..d825efeaf1 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/Configuration.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/Configuration.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; @Data @ToString @@ -23,7 +24,12 @@ @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2", "EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC"}) public class Configuration { - private List categories = new ArrayList<>(); private List nodes = new ArrayList<>(); private List connections; + private String description; + private Long timeout; + private Long publishedAt; + private Integer revision; + /** Job settings declared by the executor, such as batch size or parallelism. */ + private Map settings; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeConfiguration.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeConfiguration.java index e20f9a13bf..2b74b84169 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeConfiguration.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeConfiguration.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; @Data @ToString @@ -33,4 +34,5 @@ public class NodeConfiguration private String description; private List ports; private Map data; + private Set supportExecutors; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeItemConfiguration.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeItemConfiguration.java index 47783e467f..88c5e59514 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeItemConfiguration.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/itransient/configuration/NodeItemConfiguration.java @@ -11,6 +11,7 @@ import org.devlive.metaforge.common.view.EntityView; import java.util.List; +import java.util.Map; @Data @ToString @@ -31,4 +32,17 @@ public class NodeItemConfiguration private Object value; private List rules; private List options; + private Double min; + private Double max; + private Double step; + /** Editor tab of the field: basic, sql or advanced. Derived from the field when absent. */ + private String group; + /** + * Fields of the same node this field reads its choices from, such as {@code {source: source}} for + * a database list or {@code {source: source, database: database, table: table}} for target columns. + * Changing one of them clears this field. + */ + private Map from; + /** Fields of the upstream node supplying incoming columns, such as {@code {source: source, query: query}}. */ + private Map upstream; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationChannels.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationChannels.java new file mode 100644 index 0000000000..20f23652ed --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationChannels.java @@ -0,0 +1,106 @@ +package org.devlive.metaforge.service.notify; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.devlive.metaforge.notify.NotifyField; +import org.devlive.metaforge.notify.NotifyFieldType; +import org.devlive.metaforge.notify.NotifyService; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * The channels a user can be notified on: the built-in site channel plus every installed and + * enabled notify plugin, each with the settings it asks every user for. + */ +@Component +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "The plugin manager is a Spring-managed singleton") +public class NotificationChannels +{ + public static final String INTERNAL = "Internal"; + + private final PluginManager plugins; + + public NotificationChannels(PluginManager plugins) + { + this.plugins = plugins; + } + + @Data + @AllArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Channel + { + private String type; + private boolean builtin; + private List fields; + } + + @Data + @AllArgsConstructor + public static class Field + { + private String name; + private String label; + private String type; + private boolean required; + private String description; + private String placeholder; + + public boolean isSecret() + { + return NotifyFieldType.SECRET.name().equals(type); + } + } + + public List list() + { + List channels = new ArrayList<>(); + channels.add(new Channel(INTERNAL, true, List.of())); + plugins.getPluginInfos().stream() + .filter(info -> info.getType() == PluginType.NOTIFY) + .map(PluginMetadata::getName) + .filter(Objects::nonNull) + .distinct() + .sorted() + .forEach(name -> service(name).ifPresent(service -> channels.add(new Channel(name, false, fields(service))))); + return channels; + } + + public Optional find(String type) + { + return list().stream().filter(channel -> channel.getType().equals(type)).findFirst(); + } + + /** The plugin behind a channel, absent for the site channel and for uninstalled or disabled plugins. */ + public Optional service(String type) + { + if (type == null || INTERNAL.equals(type)) { + return Optional.empty(); + } + return plugins.getPlugin(type).flatMap(plugin -> Optional.ofNullable(plugin.getService(NotifyService.class))); + } + + private static List fields(NotifyService service) + { + List declared; + try { + declared = service.fields(); + } + catch (RuntimeException | LinkageError e) { + // A plugin built against the older interface declares no settings. + declared = List.of(); + } + return declared.stream() + .map(field -> new Field(field.getName(), field.getLabel(), field.getType().name(), field.getRequired(), field.getDescription(), field.getPlaceholder())) + .collect(Collectors.toList()); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationDispatcher.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationDispatcher.java new file mode 100644 index 0000000000..959ef5de52 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationDispatcher.java @@ -0,0 +1,158 @@ +package org.devlive.metaforge.service.notify; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.notify.model.NotifyRequest; +import org.devlive.metaforge.notify.model.NotifyResponse; +import org.devlive.metaforge.service.entity.NotificationEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEvent; +import org.devlive.metaforge.service.itransient.NotificationTypeEntity; +import org.devlive.metaforge.service.repository.NotificationRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.springframework.stereotype.Component; + +import javax.annotation.PreDestroy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Delivers one notification to every channel its user subscribed to the scene: the site channel + * stores it, plugin channels send it with the user's own settings off the calling thread. Failures + * are logged by outcome only, never with the user's settings, and never reach the caller. + */ +@Slf4j +@Component +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") +public class NotificationDispatcher +{ + private final UserRepository users; + private final NotificationRepository notifications; + private final NotificationChannels channels; + private final ExecutorService senders; + + public NotificationDispatcher(UserRepository users, NotificationRepository notifications, NotificationChannels channels) + { + this.users = users; + this.notifications = notifications; + this.channels = channels; + AtomicInteger counter = new AtomicInteger(); + this.senders = Executors.newFixedThreadPool(2, runnable -> { + Thread thread = new Thread(runnable, "notify-sender-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + } + + @PreDestroy + public void close() + { + senders.shutdown(); + } + + /** The channels of this user that receive the scene. */ + public static List subscribed(UserEntity user, NotificationScene scene) + { + List types = new ArrayList<>(); + if (user == null || scene == null) { + return types; + } + if (scene.isAdmin() && !UserNotifySettings.isAdmin(user)) { + return types; + } + for (NotificationTypeEntity entry : Optional.ofNullable(user.getNotifyConfigure()).orElse(List.of())) { + if (entry != null && entry.isEnabled() && entry.getType() != null && !types.contains(entry.getType()) + && NotificationScene.normalize(entry.getServices()).contains(scene.name())) { + types.add(entry.getType()); + } + } + return types; + } + + public void dispatch(NotificationEvent event) + { + if (event == null || event.getUser() == null || event.getScene() == null) { + return; + } + Optional found = Optional.ofNullable(event.getUser().getCode()).flatMap(users::findByCode); + if (found.isEmpty()) { + return; + } + UserEntity user = found.get(); + if (user.getRoles() == null) { + user.setRoles(event.getUser().getRoles()); + } + for (String type : subscribed(user, event.getScene())) { + if (NotificationChannels.INTERNAL.equals(type)) { + try { + saveInternal(user, event.getScene(), event.getTitle(), event.getContent(), event); + } + catch (RuntimeException e) { + log.warn("Saving site notification {} failed: {}", event.getScene(), e.getClass().getSimpleName()); + } + continue; + } + Map configure = user.getNotifyConfigure().stream() + .filter(entry -> entry != null && type.equals(entry.getType())) + .findFirst() + .map(NotificationTypeEntity::getConfigure) + .orElse(Map.of()); + send(type, event, configure == null ? Map.of() : configure); + } + } + + /** Stores a site notification; entity details come from the event when there is one. */ + public void saveInternal(UserEntity user, NotificationScene scene, String title, String content, NotificationEvent event) + { + EntityType entityType = event == null ? null : event.getEntityType(); + notifications.save(NotificationEntity.builder() + .user(user) + .userId(user.getId()) + .type(scene == null ? "TEST" : scene.name()) + .name(StringUtils.abbreviate(StringUtils.defaultString(title), 255)) + .content(StringUtils.abbreviate(StringUtils.defaultString(content), 2000)) + .entityType(entityType) + .entityCode(event == null ? null : event.getEntityCode()) + .entityName(event == null ? null : StringUtils.abbreviate(event.getEntityName(), 255)) + .entityExists(event != null && event.isEntityExists() && event.getEntityCode() != null) + .isRead(false) + .build()); + } + + private void send(String type, NotificationEvent event, Map configure) + { + NotifyRequest request = new NotifyRequest(); + request.setTitle(event.getTitle()); + request.setContent(StringUtils.isBlank(event.getContent()) ? event.getTitle() : event.getContent()); + request.setConfigure(configure); + try { + senders.submit(() -> { + try { + NotifyResponse response = channels.service(type).map(service -> service.send(request)).orElse(null); + if (response == null) { + log.debug("Notification channel {} is not available", type); + } + else if (!Boolean.TRUE.equals(response.getSuccessful())) { + log.warn("Notification {} on {} failed: {}", event.getScene(), type, response.getMessage()); + } + } + catch (RuntimeException | LinkageError e) { + log.warn("Notification {} on {} failed: {}", event.getScene(), type, e.getClass().getSimpleName()); + } + }); + } + catch (RejectedExecutionException e) { + log.debug("Notification {} on {} skipped while shutting down", event.getScene(), type); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationInbox.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationInbox.java new file mode 100644 index 0000000000..83cfaeb34b --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/NotificationInbox.java @@ -0,0 +1,226 @@ +package org.devlive.metaforge.service.notify; + +import com.fasterxml.jackson.annotation.JsonFormat; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.springframework.data.relational.core.dialect.Dialect; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * One user's site notifications: filtered pages, counts, read state and removal. Every statement is + * limited to the given user, so nobody reads or changes another user's notifications by code. + */ +@Component +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") +public class NotificationInbox +{ + static final int MAX_PAGE_SIZE = 100; + private static final String TABLE = "metaforge_notification"; + private static final char ESCAPE = '!'; + private static final String COLUMNS = "code, name, content, type, is_read, create_time, entity_type, entity_code, entity_name, entity_exists"; + + private final NamedParameterJdbcOperations jdbc; + private final Dialect dialect; + + public NotificationInbox(NamedParameterJdbcOperations jdbc, Dialect dialect) + { + this.jdbc = jdbc; + this.dialect = dialect; + } + + /** What to list: read state (null for all), a category (null for all), words in the title or content. */ + @Data + @NoArgsConstructor + public static class Query + { + private Boolean read; + private String category; + private String keyword; + private Integer page = 1; + private Integer size = 20; + } + + @Data + @NoArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Item + { + private String code; + private String title; + private String content; + private String scene; + private String category; + private boolean read; + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + private String entityType; + private String entityCode; + private String entityName; + private boolean entityExists; + } + + @Data + @NoArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Page + { + private List items = new ArrayList<>(); + private long total; + private int page; + private int size; + } + + @Data + @NoArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Summary + { + private long total; + private long unread; + /** Totals by category, system first, then the scene categories in their order. */ + private Map categories = new LinkedHashMap<>(); + } + + public Page page(Long userId, Query query) + { + MapSqlParameterSource parameters = new MapSqlParameterSource("user", userId); + String where = where(query, parameters); + int size = Math.min(MAX_PAGE_SIZE, Math.max(1, query.getSize() == null ? 20 : query.getSize())); + int page = Math.max(1, query.getPage() == null ? 1 : query.getPage()); + Page result = new Page(); + Long total = jdbc.queryForObject("SELECT COUNT(*) FROM " + TABLE + where, parameters, Long.class); + result.setTotal(total == null ? 0 : total); + result.setPage(page); + result.setSize(size); + String sql = "SELECT " + COLUMNS + " FROM " + TABLE + where + " ORDER BY create_time DESC, id DESC " + + dialect.limit().getLimitOffset(size, (long) (page - 1) * size); + result.setItems(jdbc.query(sql, parameters, (row, index) -> item(row))); + return result; + } + + public Optional find(Long userId, String code) + { + MapSqlParameterSource parameters = new MapSqlParameterSource("user", userId).addValue("code", code); + return jdbc.query("SELECT " + COLUMNS + " FROM " + TABLE + " WHERE user_id = :user AND code = :code", parameters, (row, index) -> item(row)) + .stream().findFirst(); + } + + public Summary summary(Long userId) + { + Summary summary = new Summary(); + summary.getCategories().put(NotificationScene.SYSTEM, 0L); + for (NotificationScene.Category category : NotificationScene.Category.values()) { + summary.getCategories().put(category.name(), 0L); + } + jdbc.query("SELECT type, is_read, COUNT(*) AS total FROM " + TABLE + " WHERE user_id = :user GROUP BY type, is_read", + new MapSqlParameterSource("user", userId), row -> { + long count = row.getLong("total"); + summary.setTotal(summary.getTotal() + count); + if (!row.getBoolean("is_read")) { + summary.setUnread(summary.getUnread() + count); + } + summary.getCategories().merge(NotificationScene.categoryOf(row.getString("type")), count, Long::sum); + }); + return summary; + } + + public long unread(Long userId) + { + Long count = jdbc.queryForObject("SELECT COUNT(*) FROM " + TABLE + " WHERE user_id = :user AND is_read = :unread", + new MapSqlParameterSource("user", userId).addValue("unread", false), Long.class); + return count == null ? 0 : count; + } + + /** Marks one notification read or unread; false when the user has no such notification. */ + public boolean mark(Long userId, String code, boolean read) + { + return jdbc.update("UPDATE " + TABLE + " SET is_read = :read WHERE user_id = :user AND code = :code", + new MapSqlParameterSource("user", userId).addValue("code", code).addValue("read", read)) > 0; + } + + /** Marks every unread notification read, or only those of one category; returns how many changed. */ + public int markAllRead(Long userId, String category) + { + Query query = new Query(); + query.setRead(false); + query.setCategory(category); + MapSqlParameterSource parameters = new MapSqlParameterSource("user", userId).addValue("done", true); + return jdbc.update("UPDATE " + TABLE + " SET is_read = :done" + where(query, parameters), parameters); + } + + public boolean delete(Long userId, String code) + { + return jdbc.update("DELETE FROM " + TABLE + " WHERE user_id = :user AND code = :code", + new MapSqlParameterSource("user", userId).addValue("code", code)) > 0; + } + + /** Removes every read notification of the user; returns how many went. */ + public int deleteRead(Long userId) + { + return jdbc.update("DELETE FROM " + TABLE + " WHERE user_id = :user AND is_read = :read", + new MapSqlParameterSource("user", userId).addValue("read", true)); + } + + private static String where(Query query, MapSqlParameterSource parameters) + { + List where = new ArrayList<>(); + where.add("user_id = :user"); + if (query.getRead() != null) { + where.add("is_read = :isRead"); + parameters.addValue("isRead", query.getRead()); + } + String category = query.getCategory(); + if (category != null && !category.isBlank()) { + if (NotificationScene.SYSTEM.equals(category)) { + where.add("(type IS NULL OR type NOT IN (:known))"); + parameters.addValue("known", NotificationScene.knownTypes()); + } + else { + NotificationScene.Category scenes = NotificationScene.Category.valueOf(category); + where.add("type IN (:types)"); + parameters.addValue("types", NotificationScene.typesOf(scenes)); + } + } + if (query.getKeyword() != null && !query.getKeyword().isBlank()) { + String escaped = query.getKeyword().trim().toLowerCase(Locale.ROOT) + .replace(String.valueOf(ESCAPE), "" + ESCAPE + ESCAPE) + .replace("%", ESCAPE + "%") + .replace("_", ESCAPE + "_"); + where.add("(LOWER(name) LIKE :keyword ESCAPE '" + ESCAPE + "' OR LOWER(content) LIKE :keyword ESCAPE '" + ESCAPE + "')"); + parameters.addValue("keyword", "%" + escaped + "%"); + } + return " WHERE " + String.join(" AND ", where); + } + + private static Item item(ResultSet row) + throws SQLException + { + Item item = new Item(); + item.setCode(row.getString("code")); + item.setTitle(row.getString("name")); + item.setContent(row.getString("content")); + item.setScene(row.getString("type")); + item.setCategory(NotificationScene.categoryOf(row.getString("type"))); + item.setRead(row.getBoolean("is_read")); + item.setCreateTime(row.getTimestamp("create_time")); + item.setEntityType(row.getString("entity_type")); + item.setEntityCode(row.getString("entity_code")); + item.setEntityName(row.getString("entity_name")); + item.setEntityExists(row.getBoolean("entity_exists")); + return item; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/UserNotifySettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/UserNotifySettings.java new file mode 100644 index 0000000000..483cc3ef40 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/notify/UserNotifySettings.java @@ -0,0 +1,263 @@ +package org.devlive.metaforge.service.notify; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.notify.model.NotifyRequest; +import org.devlive.metaforge.notify.model.NotifyResponse; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.itransient.NotificationTypeEntity; +import org.devlive.metaforge.service.repository.UserRepository; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A user's notification settings: which channels are on, each channel's own settings, and the + * scenes each enabled channel receives. Only enabled channels hold subscriptions, and secret + * settings are write-only: reads say whether they are set, an empty value on save keeps them. + */ +@Slf4j +@Service +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") +public class UserNotifySettings +{ + static final int MAX_VALUE_LENGTH = 2000; + + private final UserRepository users; + private final NotificationChannels channels; + private final NotificationDispatcher dispatcher; + + public UserNotifySettings(UserRepository users, NotificationChannels channels, NotificationDispatcher dispatcher) + { + this.users = users; + this.channels = channels; + this.dispatcher = dispatcher; + } + + /** One channel as the settings page sees it. */ + @Data + @NoArgsConstructor + @AllArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain request and response object") + public static class ChannelSetting + { + private String type; + private boolean enabled; + private Set services; + private Map configure; + /** Secret fields that hold a value; their values are never sent. */ + private Set configured; + } + + @Data + @AllArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Catalog + { + private List channels; + private List categories; + } + + @Data + @AllArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object") + public static class Category + { + private String key; + private List scenes; + } + + /** Channels and the scene categories this user may subscribe to. */ + public Catalog catalog(UserEntity user) + { + boolean admin = isAdmin(user); + List categories = Arrays.stream(NotificationScene.Category.values()) + .map(category -> new Category(category.name(), Arrays.stream(NotificationScene.values()) + .filter(scene -> scene.getCategory() == category && (admin || !scene.isAdmin())) + .map(Enum::name) + .collect(Collectors.toList()))) + .filter(category -> !category.getScenes().isEmpty()) + .collect(Collectors.toList()); + return new Catalog(channels.list(), categories); + } + + /** The signed-in user's settings as stored, secrets masked. */ + public CommonResponse> settings(UserEntity login) + { + return users.findByCode(login.getCode()) + .map(user -> { + user.setRoles(login.getRoles()); + return CommonResponse.success(read(user)); + }) + .orElseGet(() -> CommonResponse.failure("User not found")); + } + + List read(UserEntity user) + { + Map stored = stored(user); + Set allowed = allowedScenes(user); + return channels.list().stream().map(channel -> { + NotificationTypeEntity saved = stored.get(channel.getType()); + Map values = saved == null || saved.getConfigure() == null ? Map.of() : saved.getConfigure(); + Map configure = new LinkedHashMap<>(); + Set configured = new LinkedHashSet<>(); + for (NotificationChannels.Field field : channel.getFields()) { + String value = values.get(field.getName()); + if (field.isSecret()) { + configure.put(field.getName(), ""); + if (StringUtils.isNotBlank(value)) { + configured.add(field.getName()); + } + } + else { + configure.put(field.getName(), StringUtils.defaultString(value)); + } + } + boolean enabled = saved != null && saved.isEnabled(); + Set services = enabled ? NotificationScene.normalize(saved.getServices()) : new LinkedHashSet<>(); + services.retainAll(allowed); + return new ChannelSetting(channel.getType(), enabled, services, configure, configured); + }).collect(Collectors.toList()); + } + + public CommonResponse> save(UserEntity login, List request) + { + Optional found = users.findByCode(login.getCode()); + if (found.isEmpty()) { + return CommonResponse.failure("User not found"); + } + UserEntity user = found.get(); + // Roles are not stored on the user row; the signed-in principal carries them. + user.setRoles(login.getRoles()); + Map stored = stored(user); + Set allowed = allowedScenes(login); + Map incoming = new LinkedHashMap<>(); + for (ChannelSetting setting : Optional.ofNullable(request).orElse(List.of())) { + if (setting == null || setting.getType() == null) { + continue; + } + if (channels.find(setting.getType()).isEmpty()) { + return CommonResponse.failure("Unknown notification channel: " + setting.getType()); + } + incoming.put(setting.getType(), setting); + } + for (ChannelSetting setting : incoming.values()) { + NotificationChannels.Channel channel = channels.find(setting.getType()).orElseThrow(); + NotificationTypeEntity previous = stored.get(channel.getType()); + Map before = previous == null || previous.getConfigure() == null ? Map.of() : previous.getConfigure(); + Map values = new LinkedHashMap<>(); + Map given = Optional.ofNullable(setting.getConfigure()).orElse(Map.of()); + for (NotificationChannels.Field field : channel.getFields()) { + String value = StringUtils.trimToEmpty(given.get(field.getName())); + // Secrets are never sent back to the page: leaving one empty keeps the stored value. + if (field.isSecret() && value.isEmpty()) { + value = StringUtils.defaultString(before.get(field.getName())); + } + if (value.length() > MAX_VALUE_LENGTH) { + return CommonResponse.failure(channel.getType() + ": " + field.getLabel() + " is too long"); + } + if ("URL".equals(field.getType()) && !value.isEmpty() && !value.matches("(?i)^https?://\\S+$")) { + return CommonResponse.failure(channel.getType() + ": " + field.getLabel() + " must be an http(s) address"); + } + if (setting.isEnabled() && field.isRequired() && value.isEmpty()) { + return CommonResponse.failure(channel.getType() + ": " + field.getLabel() + " is required"); + } + if (!value.isEmpty()) { + values.put(field.getName(), value); + } + } + Set services = new LinkedHashSet<>(); + // A disabled channel receives nothing, so it keeps its settings but no subscriptions. + if (setting.isEnabled()) { + Set requested = Optional.ofNullable(setting.getServices()).orElse(Set.of()); + Set unknown = requested.stream().filter(scene -> !allowed.contains(scene)).collect(Collectors.toSet()); + if (!unknown.isEmpty()) { + return CommonResponse.failure("Unknown notification scenes: " + String.join(", ", unknown)); + } + services.addAll(requested); + } + stored.put(channel.getType(), NotificationTypeEntity.builder() + .type(channel.getType()).enabled(setting.isEnabled()).services(services).configure(values).build()); + } + user.setNotifyConfigure(new ArrayList<>(stored.values())); + users.save(user); + return CommonResponse.success(read(user)); + } + + /** Sends a test message with the given settings; empty secrets fall back to the stored ones. */ + public CommonResponse test(UserEntity login, ChannelSetting setting) + { + if (setting == null || setting.getType() == null) { + return CommonResponse.failure("Unknown notification channel"); + } + Optional channel = channels.find(setting.getType()); + Optional user = users.findByCode(login.getCode()); + if (channel.isEmpty() || user.isEmpty()) { + return CommonResponse.failure("Unknown notification channel: " + setting.getType()); + } + if (channel.get().isBuiltin()) { + dispatcher.saveInternal(user.get(), null, "通知测试", "这是一条来自 MetaForge 的测试通知。", null); + return CommonResponse.success("sent"); + } + NotificationTypeEntity previous = stored(user.get()).get(setting.getType()); + Map before = previous == null || previous.getConfigure() == null ? Map.of() : previous.getConfigure(); + Map values = new LinkedHashMap<>(); + for (NotificationChannels.Field field : channel.get().getFields()) { + String value = StringUtils.trimToEmpty(Optional.ofNullable(setting.getConfigure()).orElse(Map.of()).get(field.getName())); + values.put(field.getName(), value.isEmpty() && field.isSecret() ? StringUtils.defaultString(before.get(field.getName())) : value); + } + NotifyRequest request = new NotifyRequest(); + request.setTitle("MetaForge 通知测试"); + request.setContent("这是一条来自 MetaForge 的测试通知,收到说明 " + setting.getType() + " 渠道配置正确。"); + request.setConfigure(values); + try { + NotifyResponse response = channels.service(setting.getType()).map(service -> service.send(request)).orElse(null); + if (response == null) { + return CommonResponse.failure("The notification plugin is not available"); + } + return response.getSuccessful() ? CommonResponse.success("sent") : CommonResponse.failure(StringUtils.defaultIfBlank(response.getMessage(), "Sending failed")); + } + catch (RuntimeException e) { + log.warn("Test notification on {} failed: {}", setting.getType(), e.getClass().getSimpleName()); + return CommonResponse.failure("Sending failed: " + StringUtils.defaultString(e.getMessage(), e.getClass().getSimpleName())); + } + } + + static boolean isAdmin(UserEntity user) + { + return user != null && user.getRoles() != null && user.getRoles().stream() + .anyMatch(role -> role != null && role.getCode() != null && role.getCode().contains("ADMIN")); + } + + private static Set allowedScenes(UserEntity user) + { + boolean admin = isAdmin(user); + return Arrays.stream(NotificationScene.values()).filter(scene -> admin || !scene.isAdmin()).map(Enum::name) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private static Map stored(UserEntity user) + { + Map stored = new LinkedHashMap<>(); + for (NotificationTypeEntity entry : Optional.ofNullable(user.getNotifyConfigure()).orElse(List.of())) { + if (entry != null && entry.getType() != null && !stored.containsKey(entry.getType())) { + stored.put(entry.getType(), entry); + } + } + return stored; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ConfigureRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ConfigureRepository.java index 1fa8e8a41f..9e4238fce6 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ConfigureRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ConfigureRepository.java @@ -9,7 +9,7 @@ public interface ConfigureRepository extends PagingAndSortingRepository { - /** 按范畴 + 名称定位唯一一条配置(如 EXECUTOR / Local) */ + /** 按范畴 + 名称定位唯一一条配置(如 EXECUTOR / MetaForge) */ Optional findByCategoryAndName(String category, String name); /** 列出某范畴下的全部配置(系统配置页面用) */ diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/DataSetRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/DataSetRepository.java index 499c8dff4e..3ac9120906 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/DataSetRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/DataSetRepository.java @@ -18,6 +18,8 @@ default Visibility visibleTo(UserEntity user) return Visibility.where("userId", user.getId()); } + long countByUserId(Long userId); + Page findAllByUserId(Long userId, Pageable pageable); List findAllBySyncMode(SyncMode mode); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java index 33e893e47d..638eebfe30 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java @@ -6,6 +6,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import java.util.Optional; + public interface ReportRepository extends BaseRepository { @@ -16,4 +18,6 @@ default Visibility visibleTo(UserEntity user) } Page findAllByUserId(Long userId, Pageable pageable); + + Optional findByShareToken(String shareToken); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserOAuthRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserOAuthRepository.java new file mode 100644 index 0000000000..93626b68a4 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserOAuthRepository.java @@ -0,0 +1,12 @@ +package org.devlive.metaforge.service.repository; + +import org.devlive.metaforge.service.entity.UserOAuthEntity; +import org.springframework.data.repository.PagingAndSortingRepository; + +import java.util.Optional; + +public interface UserOAuthRepository + extends PagingAndSortingRepository +{ + Optional findByProviderAndSubject(String provider, String subject); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java index c66fe850b8..f499ff6bb9 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java @@ -1,6 +1,9 @@ package org.devlive.metaforge.service.repository; import org.devlive.metaforge.service.entity.UserEntity; +import org.springframework.data.jdbc.repository.query.Modifying; +import org.springframework.data.jdbc.repository.query.Query; +import org.springframework.data.repository.query.Param; import java.util.Optional; @@ -9,5 +12,17 @@ public interface UserRepository { Optional findByUsername(String username); + Optional findByEmail(String email); + UserEntity findByUsernameAndSystemTrue(String username); + + /** Only the home page column changes, so a concurrent profile save cannot be overwritten. */ + @Modifying + @Query("UPDATE metaforge_user SET home_page = :homePage WHERE id = :id") + void updateHomePage(@Param("id") Long id, @Param("homePage") String homePage); + + /** A deleted data screen stops being anyone's home page. */ + @Modifying + @Query("UPDATE metaforge_user SET home_page = NULL WHERE home_page = :homePage") + void clearHomePage(@Param("homePage") String homePage); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/WorkflowRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/WorkflowRepository.java index c6de52ee36..a79362ab0c 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/WorkflowRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/WorkflowRepository.java @@ -1,8 +1,22 @@ package org.devlive.metaforge.service.repository; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.entity.WorkflowEntity; +import org.devlive.metaforge.service.persistence.Visibility; + +import java.util.List; public interface WorkflowRepository extends BaseRepository { + List findAllByUserId(Long userId); + + List findAllByState(RunState state); + + @Override + default Visibility visibleTo(UserEntity user) + { + return Visibility.where("userId", user.getId()); + } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/DashboardShareRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/DashboardShareRepository.java new file mode 100644 index 0000000000..8b437a2372 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/DashboardShareRepository.java @@ -0,0 +1,155 @@ +package org.devlive.metaforge.service.repository.jdbc; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.service.enums.SharePermission; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Sharing of data screens, read and replaced with explicit SQL. Callers run replacements inside + * their own transaction. + */ +@Repository +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class DashboardShareRepository +{ + public static final String USER = "USER"; + public static final String ROLE = "ROLE"; + + private final NamedParameterJdbcOperations jdbc; + + public DashboardShareRepository(NamedParameterJdbcOperations jdbc) + { + this.jdbc = jdbc; + } + + @Data + @NoArgsConstructor + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) + public static class Share + { + private Long dashboardId; + private String everyonePermission; + private String widgets; + private boolean publicEnabled; + private String publicToken; + private boolean publicEmbed; + private String publicOrigins; + private Date publicExpiresAt; + } + + @Data + @NoArgsConstructor + public static class Member + { + private String type; + private Long id; + private String permission; + + public Member(String type, Long id, String permission) + { + this.type = type; + this.id = id; + this.permission = permission; + } + } + + public Optional find(Long dashboardId) + { + return jdbc.query("SELECT * FROM metaforge_dashboard_share WHERE dashboard_id = :id", Map.of("id", dashboardId), (rs, row) -> { + Share share = new Share(); + share.setDashboardId(rs.getLong("dashboard_id")); + share.setEveryonePermission(rs.getString("everyone_permission")); + share.setWidgets(rs.getString("widgets")); + share.setPublicEnabled(rs.getBoolean("public_enabled")); + share.setPublicToken(rs.getString("public_token")); + share.setPublicEmbed(rs.getBoolean("public_embed")); + share.setPublicOrigins(rs.getString("public_origins")); + Timestamp expires = rs.getTimestamp("public_expires_at"); + share.setPublicExpiresAt(expires == null ? null : new Date(expires.getTime())); + return share; + }).stream().findFirst(); + } + + /** The screen behind an enabled public link with this token. */ + public Optional findPublic(String token) + { + return jdbc.queryForList("SELECT dashboard_id FROM metaforge_dashboard_share WHERE public_token = :token AND public_enabled = :enabled", + new MapSqlParameterSource("token", token).addValue("enabled", true), Long.class).stream().findFirst(); + } + + public void save(Share share) + { + jdbc.update("DELETE FROM metaforge_dashboard_share WHERE dashboard_id = :id", Map.of("id", share.getDashboardId())); + jdbc.update("INSERT INTO metaforge_dashboard_share (dashboard_id, everyone_permission, widgets, public_enabled, public_token, public_embed, " + + "public_origins, public_expires_at, update_time) VALUES (:id, :everyone, :widgets, :enabled, :token, :embed, :origins, :expires, :now)", + new MapSqlParameterSource("id", share.getDashboardId()) + .addValue("everyone", share.getEveryonePermission()) + .addValue("widgets", share.getWidgets()) + .addValue("enabled", share.isPublicEnabled()) + .addValue("token", share.getPublicToken()) + .addValue("embed", share.isPublicEmbed()) + .addValue("origins", share.getPublicOrigins()) + .addValue("expires", share.getPublicExpiresAt() == null ? null : new Timestamp(share.getPublicExpiresAt().getTime())) + .addValue("now", new Timestamp(System.currentTimeMillis()))); + } + + public List members(Long dashboardId) + { + return jdbc.query("SELECT member_type, member_id, permission FROM metaforge_dashboard_member WHERE dashboard_id = :id ORDER BY member_type, member_id", + Map.of("id", dashboardId), (rs, row) -> new Member(rs.getString("member_type"), rs.getLong("member_id"), rs.getString("permission"))); + } + + public void replaceMembers(Long dashboardId, Collection members) + { + jdbc.update("DELETE FROM metaforge_dashboard_member WHERE dashboard_id = :id", Map.of("id", dashboardId)); + for (Member member : members) { + jdbc.update("INSERT INTO metaforge_dashboard_member (dashboard_id, member_type, member_id, permission) VALUES (:id, :type, :member, :permission)", + new MapSqlParameterSource("id", dashboardId).addValue("type", member.getType()).addValue("member", member.getId()) + .addValue("permission", member.getPermission())); + } + } + + public void delete(Long dashboardId) + { + jdbc.update("DELETE FROM metaforge_dashboard_member WHERE dashboard_id = :id", Map.of("id", dashboardId)); + jdbc.update("DELETE FROM metaforge_dashboard_share WHERE dashboard_id = :id", Map.of("id", dashboardId)); + } + + /** + * Screens shared with a user directly, through one of their roles or with everyone, each with + * the highest permission any of these grants. + */ + public Map sharedWith(Long userId, Collection roleIds) + { + Map result = new LinkedHashMap<>(); + MapSqlParameterSource params = new MapSqlParameterSource("user", userId).addValue("userType", USER).addValue("roleType", ROLE) + .addValue("roles", roleIds.isEmpty() ? List.of(-1L) : new ArrayList<>(roleIds)); + jdbc.query("SELECT dashboard_id, permission FROM metaforge_dashboard_member WHERE (member_type = :userType AND member_id = :user) " + + "OR (member_type = :roleType AND member_id IN (:roles))", params, rs -> { + merge(result, rs.getLong("dashboard_id"), rs.getString("permission")); + }); + jdbc.query("SELECT dashboard_id, everyone_permission FROM metaforge_dashboard_share WHERE everyone_permission IS NOT NULL", Map.of(), rs -> { + merge(result, rs.getLong("dashboard_id"), rs.getString("everyone_permission")); + }); + return result; + } + + private static void merge(Map result, Long dashboardId, String permission) + { + SharePermission.grantable(permission).ifPresent(granted -> result.merge(dashboardId, granted, + (left, right) -> left.atLeast(right) ? left : right)); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/LinkRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/LinkRepository.java index 9eac43ab2c..f0efe38897 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/LinkRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/jdbc/LinkRepository.java @@ -37,6 +37,7 @@ public void replaceUserRoles(Long userId, Collection roleIds) public void deleteUser(Long userId) { jdbc.update("DELETE FROM metaforge_user_role WHERE user_id = :id", Map.of("id", userId)); + jdbc.update("DELETE FROM metaforge_user_oauth WHERE user_id = :id", Map.of("id", userId)); } public long countUsersOfRole(Long roleId) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/security/RunAs.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/security/RunAs.java new file mode 100644 index 0000000000..623f630bdd --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/security/RunAs.java @@ -0,0 +1,37 @@ +package org.devlive.metaforge.service.security; + +import org.devlive.metaforge.service.entity.UserEntity; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.ArrayList; +import java.util.function.Supplier; + +/** + * Runs an action as another user, so ownership checks apply as for that user, then restores the + * caller's identity. Used where an owner shares something whose reads are checked against them. + */ +public final class RunAs +{ + private RunAs() + {} + + /** Without a user the action runs anonymously, where ownership checks deny access. */ + public static T user(UserEntity user, Supplier action) + { + SecurityContext previous = SecurityContextHolder.getContext(); + SecurityContext context = SecurityContextHolder.createEmptyContext(); + if (user != null) { + UserDetailsService principal = new UserDetailsService(user.getId(), user.getCode(), user.getUsername(), "", new ArrayList<>(), null); + context.setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); + } + SecurityContextHolder.setContext(context); + try { + return action.get(); + } + finally { + SecurityContextHolder.setContext(previous); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java new file mode 100644 index 0000000000..2f37297c68 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java @@ -0,0 +1,24 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; + +import java.util.Map; + +/** Sign-in, sign-up and password reset, each feature only as the system settings allow. */ +public interface AuthService +{ + /** What the sign-in and sign-up pages offer; public. */ + Map options(); + + CommonResponse signin(AuthBody body); + + CommonResponse signup(AuthBody body); + + /** Mails a code; answers with the seconds to wait before asking again. */ + CommonResponse sendCode(EmailCodeBody body); + + CommonResponse reset(PasswordResetBody body); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ConfigureService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ConfigureService.java deleted file mode 100644 index 1e6ac233f4..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ConfigureService.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.devlive.metaforge.service.service; - -import org.devlive.metaforge.common.response.CommonResponse; -import org.devlive.metaforge.service.itransient.configuration.Configuration; - -public interface ConfigureService -{ - CommonResponse getExecutor(); -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DashboardService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DashboardService.java index 6aa2405f09..bc6abe6a0a 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DashboardService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DashboardService.java @@ -1,8 +1,44 @@ package org.devlive.metaforge.service.service; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.DashboardShareBody; import org.devlive.metaforge.service.entity.DashboardEntity; +import java.util.List; +import java.util.Map; + public interface DashboardService extends BaseService { + /** + * Opens one of the current user's published data screens as their home page, or restores the + * default home page when {@code code} is null. + */ + CommonResponse setHomePage(String code); + + /** Screens other users share with the signed-in user, each with the user's permission. */ + CommonResponse> shared(); + + CommonResponse> shared(String keyword, String sort); + + /** Refreshes a dynamic chart of a screen for anyone who may see it; reports run as the owner. */ + CommonResponse>> refreshChart(String code, String reportCode); + + /** How the owner shares a screen. */ + CommonResponse getShare(String code); + + /** Replaces how the owner shares a screen. */ + CommonResponse saveShare(String code, DashboardShareBody body); + + /** A published screen behind an enabled, unexpired public link, as anonymous visitors see it. */ + CommonResponse> publicView(String token); + + /** Refreshes a dynamic chart of a public screen; it runs as the owner. */ + CommonResponse>> publicRefresh(String token, String reportCode); + + /** The frame-ancestors sources of an embeddable public screen; empty when it may not be framed. */ + java.util.Optional publicFrameAncestors(String token); + + /** Users and roles the owner can share with, matching a keyword. */ + CommonResponse> shareCandidates(String keyword); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DataSetService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DataSetService.java index 925400e3cd..e9cdb47d91 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DataSetService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DataSetService.java @@ -16,19 +16,25 @@ public interface DataSetService extends BaseService { + CommonResponse count(); + CommonResponse preview(String code); CommonResponse rebuild(String code); CommonResponse> getColumnsByCode(String code); - CommonResponse syncData(String code); + /** + * 提交一次同步,返回新建的同步历史(进入排队)。数据集已有排队或运行中的同步时失败。 + * Submit a sync and return the new, queued history row; fails while the dataset already has one queued or running. + */ + CommonResponse syncData(String code); /** * 同步并允许调用方临时覆盖一组 tunable 字段;非 tunable 的字段会被忽略。 * Trigger sync with optional per-invocation overrides; only tunable fields are honored. */ - CommonResponse syncData(String code, java.util.Map overrides); + CommonResponse syncData(String code, java.util.Map overrides); /** * 返回当前 executor 的可调字段(tunable=true),defaultValue 已替换为该数据集对应 executor 的 effective 值, @@ -54,8 +60,38 @@ public interface DataSetService CommonResponse> getHistoryLog(Long id); /** - * 停止正在运行的同步任务。仅对 state=RUNNING 且当前进程持有该 taskName 的任务生效 - * Stop a running sync task. Effective only when state=RUNNING and the task is held by this process + * 取消同步:排队中的直接取消;运行中的请求执行器停止(仅对当前进程持有的任务生效) + * Cancel a sync: a queued one is cancelled at once, a running one is asked to stop (only on the node running it) */ CommonResponse stopHistory(Long id); + + /** + * 一次同步的实时指标、状态与状态转换;已结束的返回落库的最终结果 + * Live metrics, state and transitions of one sync; the persisted final result once it has finished + */ + CommonResponse getHistoryMetrics(Long id); + + /** + * 暂停运行中的同步,提交完已成形的批次后保存断点。只有按键分片的同步能暂停。 + * Pause a running sync; it saves a checkpoint after committing the batches already formed. + */ + CommonResponse pauseHistory(Long id); + + /** + * 当前用户未结束的同步,供列表页显示进度。 + * The signed-in user's unfinished syncs, for the progress on the list page. + */ + CommonResponse> getActiveSyncs(); + + /** + * 继续已暂停的同步,回到队列后从断点续传。 + * Resume a paused sync: back into the queue, continuing from its checkpoint. + */ + CommonResponse resumeHistory(Long id); + + /** + * 让失败、超时、停止或中断的同步再次运行,只在不会重复写入时允许。 + * Run a failed, timed-out, stopped or interrupted sync again, only when that writes nothing twice. + */ + CommonResponse retryHistory(Long id); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java new file mode 100644 index 0000000000..74f584ba2b --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java @@ -0,0 +1,28 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.exporter.ExportPlan; + +import java.util.List; +import java.util.Map; + +public interface DatabaseExportService +{ + /** Installed export formats with their file extension, version and the settings they declare. */ + CommonResponse>> formats(); + + /** The first rows of one table as read and as the chosen format writes them. */ + CommonResponse> sample(String code, String database, ExportPlan plan, String table, int rows); + + /** Checks every table of a plan and lists the files the export will write. */ + CommonResponse> check(String code, String database, ExportPlan plan); + + /** Starts the export in the background and returns its job id. */ + CommonResponse> execute(String code, String database, ExportPlan plan); + + /** Progress and result of an export job. */ + CommonResponse> job(String id); + + /** The finished file of an export job. */ + CommonResponse file(String id); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java index 081dd64569..8f770868b7 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java @@ -42,12 +42,19 @@ public void warnOnDefaultSecret() } public String generateJwtToken(Authentication authentication) + { + return generateJwtToken(authentication, 0); + } + + /** A token lasting the given milliseconds, or metaforge.security.expiration when it is 0. */ + public String generateJwtToken(Authentication authentication, long expirationMs) { UserDetailsService userPrincipal = (UserDetailsService) authentication.getPrincipal(); + long lasting = expirationMs > 0 ? expirationMs : jwtExpirationMs; return Jwts.builder() .setSubject((userPrincipal.getUsername())) .setIssuedAt(new Date()) - .setExpiration(new Date((new Date()).getTime() + jwtExpirationMs)) + .setExpiration(new Date((new Date()).getTime() + lasting)) .signWith(SignatureAlgorithm.HS512, jwtSecret) .compact(); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/NotificationService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/NotificationService.java deleted file mode 100644 index 7426d22a2d..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/NotificationService.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.devlive.metaforge.service.service; - -import org.devlive.metaforge.service.entity.NotificationEntity; - -public interface NotificationService - extends BaseService -{ -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/OAuthService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/OAuthService.java new file mode 100644 index 0000000000..176398dee4 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/OAuthService.java @@ -0,0 +1,41 @@ +package org.devlive.metaforge.service.service; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.devlive.metaforge.common.response.CommonResponse; + +/** + * Signing in with GitHub, Google or an OpenID Connect provider by the authorization code flow. + * Each provider is used only while system settings have it turned on and configured. + */ +public interface OAuthService +{ + /** Where to send the browser, and the state to keep in it when a sign-in began. */ + @Getter + @AllArgsConstructor + final class Redirect + { + private final String location; + /** Null when no sign-in began and the location is the sign-in page with an error. */ + private final String state; + } + + /** + * Starts a sign-in at the provider. + * + * @param redirect the page to return to afterwards, a path on this site + * @param base the address of this site as the request reached it, used when no site address is set + */ + Redirect authorize(String provider, String redirect, String base); + + /** + * Finishes a sign-in the provider sent back, and answers where to send the browser: the + * callback page with a ticket for the token, or with an error code. + * + * @param browserState the state kept in the browser when the sign-in began + */ + String callback(String provider, String code, String state, String browserState, String error, String base); + + /** Exchanges a ticket from {@link #callback} for the token, once. */ + CommonResponse redeem(String ticket); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/PluginAuditService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/PluginAuditService.java index 22631bc24a..ef3ee0ef1f 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/PluginAuditService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/PluginAuditService.java @@ -2,6 +2,7 @@ import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.activity.HeatmapActivity; +import org.devlive.metaforge.service.activity.QueryActivity; import org.devlive.metaforge.service.entity.PluginAuditEntity; import org.devlive.metaforge.service.itransient.ContributionRadar; import org.devlive.metaforge.spi.model.Response; @@ -17,6 +18,8 @@ public interface PluginAuditService CommonResponse> getContributionRadar(); + CommonResponse getHomeActivity(int days); + CommonResponse getById(Long id); CommonResponse getData(String code); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportService.java index 02d4a76dc6..c30fdf5688 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportService.java @@ -5,8 +5,13 @@ import org.devlive.metaforge.service.entity.PageEntity; import org.devlive.metaforge.service.entity.ReportEntity; +import java.util.List; +import java.util.Map; + public interface ReportService extends BaseService { CommonResponse> getAll(FilterBody filter); + + CommonResponse>> refreshChart(String code); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java new file mode 100644 index 0000000000..a7c4e8215f --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java @@ -0,0 +1,117 @@ +package org.devlive.metaforge.service.service; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.common.response.CommonResponse; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Public links and iframe embedding of reports. The share token is the only public identifier; + * public reads run the report's saved definition as its owner and never accept SQL or a query + * from the visitor. + */ +public interface ReportShareService +{ + /** Owner only: turn sharing on or off, allow embedding, restrict framing origins, rotate the link. */ + CommonResponse share(String code, ShareRequest request); + + /** The public view of a shared report, without its SQL, data source or owner. */ + CommonResponse view(String token); + + /** Fresh rows of a shared dynamic query chart. */ + CommonResponse>> refresh(String token); + + /** Rows of a shared dataset report, computed from the saved dataset query. */ + CommonResponse data(String token); + + /** + * The CSP frame-ancestors value for an embeddable shared report, or empty when the report + * cannot be embedded. + */ + Optional frameAncestors(String token); + + @Data + @NoArgsConstructor + @AllArgsConstructor + class ShareRequest + { + private boolean enabled; + private boolean embed; + private String origins; + private boolean rotate; + /** Epoch milliseconds after which the link stops working; null never expires. */ + private Long expiresAt; + /** Show the report name and the description on the public page; null keeps them shown. */ + private Boolean showTitle; + private Boolean showDescription; + /** Offer the data as a table, and as a CSV download; null keeps them off. */ + private Boolean allowData; + private Boolean allowDownload; + + public ShareRequest(boolean enabled, boolean embed, String origins, boolean rotate) + { + this(enabled, embed, origins, rotate, null, null, null, null, null); + } + + public ShareRequest(boolean enabled, boolean embed, String origins, boolean rotate, Long expiresAt, Boolean showTitle, Boolean showDescription) + { + this(enabled, embed, origins, rotate, expiresAt, showTitle, showDescription, null, null); + } + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + class ShareState + { + private boolean enabled; + private boolean embed; + private String origins; + private String token; + private Long expiresAt; + private boolean showTitle; + private boolean showDescription; + private boolean allowData; + private boolean allowDownload; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + class PublicReport + { + private String name; + private String description; + private String type; + private boolean realtime; + private String configure; + private boolean embed; + private boolean showTitle; + private boolean showDescription; + private boolean allowData; + private boolean allowDownload; + } + + /** The report data as a table: column names and rows in the same order. */ + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}, justification = "Plain response object, built once per request") + @Data + @NoArgsConstructor + @AllArgsConstructor + class Table + { + private List headers; + private List> rows; + private boolean truncated; + } + + /** The data table, when the owner offers it on the public page. */ + CommonResponse table(String token); + + /** The report name and data for a CSV download, when the owner allows downloads. */ + Optional> download(String token); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java new file mode 100644 index 0000000000..53f5fb546a --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java @@ -0,0 +1,57 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.FsSettings; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.OAuthProvider; +import org.devlive.metaforge.service.settings.OAuthSettings; + +import java.util.Map; + +/** Settings an administrator changes in system settings: sign-in and registration features, the mail server and third-party sign-in. */ +public interface SystemSettingService +{ + AuthSettings auth(); + + MailSettings mail(); + + OAuthSettings oauth(OAuthProvider provider); + + /** Where uploads and query results are stored. */ + FsSettings fs(); + + /** Every group for the settings page, the mail password and client secrets left out. */ + Map settings(); + + /** + * Saves the groups; an empty mail password or client secret keeps the stored one. + * + * @param oauth the providers by id (github, google, oidc); providers left out, or all of them when null, stay as they are + */ + default CommonResponse> save(AuthSettings auth, MailSettings mail, Map oauth) + { + return save(auth, mail, oauth, null); + } + + /** + * Saves the groups as {@link #save(AuthSettings, MailSettings, Map)} does, and file storage when + * it is passed; an empty secret keeps the stored one. + */ + CommonResponse> save(AuthSettings auth, MailSettings mail, Map oauth, FsSettings fs); + + /** Writes and deletes a probe file with the given storage, an empty secret meaning the saved one. */ + CommonResponse testFs(FsSettings fs); + + /** + * Tries runtime settings before they are saved: an executor plugin checks its values, and the + * dataset target connects with them. + * + * @param category EXECUTOR or DATASET + * @param name the executor plugin's name, or the dataset target's + */ + CommonResponse testRuntime(String category, String name, Map values); + + /** Sends a test mail with the saved server, or with the given one when it is passed. */ + CommonResponse testMail(MailSettings mail, String to); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java index fcd7b35a7c..189b7ea717 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java @@ -20,7 +20,18 @@ public interface UserService { CommonResponse saveOrUpdate(UserEntity configure); - CommonResponse authenticate(UserEntity configure); + /** + * Signs in and returns the token. + * + * @param expiration how long the token lasts in milliseconds; 0 for the configured default + */ + CommonResponse authenticate(UserEntity configure, long expiration); + + /** + * Signs in a user whose identity was checked some other way, such as by a third-party sign-in + * provider, and returns the same token as {@link #authenticate(UserEntity, long)}. + */ + CommonResponse issue(String username, long expiration); CommonResponse info(String code); @@ -57,6 +68,4 @@ public interface UserService CommonResponse changeEditorConfigure(UserEditorEntity configure); CommonResponse uploadAvatar(UploadBody configure); - - CommonResponse changeNotify(UserEntity configure); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/WorkflowService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/WorkflowService.java index c1c7deca47..2117b19884 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/WorkflowService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/WorkflowService.java @@ -2,8 +2,10 @@ import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.entity.WorkflowEntity; +import org.devlive.metaforge.service.workflow.WorkflowRun; import java.util.List; +import java.util.Map; public interface WorkflowService extends BaseService @@ -13,4 +15,24 @@ public interface WorkflowService CommonResponse> log(String code); CommonResponse restart(String code); + + CommonResponse> catalog(); + + CommonResponse> validate(WorkflowEntity workflow); + + CommonResponse> runs(String code); + + CommonResponse run(String code, String runId); + + CommonResponse> log(String code, String runId); + + CommonResponse> statistics(); + + CommonResponse publish(String code); + + /** Column names and types a query on a saved data source returns, without reading rows. */ + CommonResponse>> queryColumns(String source, String query); + + /** Columns and the first rows of a query on a saved data source, for previewing it in the editor. */ + CommonResponse> queryPreview(String source, String query); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java new file mode 100644 index 0000000000..ce1c6e4ce2 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java @@ -0,0 +1,263 @@ +package org.devlive.metaforge.service.service.impl; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.captcha.entity.ResultEntity; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.AuthService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.EmailCodes; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.OAuthProvider; +import org.devlive.metaforge.service.settings.OAuthSettings; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; + +import javax.mail.MessagingException; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +@Slf4j +@Service +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class AuthServiceImpl + implements AuthService +{ + private static final Pattern EMAIL = Pattern.compile("^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$"); + private static final Pattern USERNAME = Pattern.compile("^[A-Za-z0-9_]{3,32}$"); + private static final int PASSWORD_MIN = 6; + private static final int PASSWORD_MAX = 120; + + private final SystemSettingService settings; + private final EmailCodes codes; + private final MailService mailService; + private final UserService userService; + private final UserRepository userRepository; + private final PasswordEncoder encoder; + private final InitializerConfigure initializer; + + public AuthServiceImpl(SystemSettingService settings, EmailCodes codes, MailService mailService, UserService userService, UserRepository userRepository, + PasswordEncoder encoder, InitializerConfigure initializer) + { + this.settings = settings; + this.codes = codes; + this.mailService = mailService; + this.userService = userService; + this.userRepository = userRepository; + this.encoder = encoder; + this.initializer = initializer; + } + + static boolean isEmail(String value) + { + return value != null && value.length() <= 255 && EMAIL.matcher(value.trim()).matches(); + } + + private static String normalize(String email) + { + return email.trim().toLowerCase(Locale.ROOT); + } + + /** Email verification and password reset only count once mail can be sent. */ + private static boolean verification(AuthSettings auth, MailSettings mail) + { + return auth.isRegistration() && auth.isEmailVerification() && mail.ready(); + } + + private static boolean reset(AuthSettings auth, MailSettings mail) + { + return auth.isPasswordReset() && mail.ready(); + } + + @Override + public Map options() + { + AuthSettings auth = settings.auth(); + MailSettings mail = settings.mail(); + Map options = new LinkedHashMap<>(); + options.put("registration", auth.isRegistration()); + options.put("captcha", auth.isCaptcha()); + options.put("rememberMe", auth.isRememberMe()); + options.put("rememberDays", auth.getRememberDays()); + options.put("emailSignin", auth.isEmailSignin()); + options.put("emailVerification", verification(auth, mail)); + options.put("passwordReset", reset(auth, mail)); + options.put("termsUrl", auth.getTermsUrl()); + options.put("privacyUrl", auth.getPrivacyUrl()); + // Only the providers that are on and fully configured; the others are not shown at all. + List> providers = new ArrayList<>(); + for (OAuthProvider provider : OAuthProvider.values()) { + OAuthSettings configured = settings.oauth(provider); + if (configured.ready()) { + providers.add(Map.of("id", provider.getId(), "name", StringUtils.defaultIfBlank(configured.getName(), provider.getLabel()))); + } + } + options.put("providers", providers); + return options; + } + + private CommonResponse captcha(AuthSettings auth, AuthBody body) + { + if (!auth.isCaptcha()) { + return null; + } + if (ObjectUtils.isEmpty(body.getCaptcha()) || body.getTimestamp() == null) { + return CommonResponse.failure("Captcha must not be empty"); + } + ResultEntity entity = initializer.getCache().getIfPresent(body.getTimestamp()); + if (entity == null) { + return CommonResponse.failure("Captcha does not exists"); + } + if (!entity.getResult().equals(body.getCaptcha())) { + initializer.getCache().invalidate(body.getTimestamp()); + return CommonResponse.failure("The calculation result is incorrect"); + } + return null; + } + + @Override + public CommonResponse signin(AuthBody body) + { + AuthSettings auth = settings.auth(); + CommonResponse failed = captcha(auth, body); + if (failed != null) { + return failed; + } + String username = body.getUsername(); + if (auth.isEmailSignin() && isEmail(username)) { + Optional owner = userRepository.findByEmail(normalize(username)); + if (owner.isPresent()) { + username = owner.get().getUsername(); + } + } + long days = auth.isRememberMe() && Boolean.TRUE.equals(body.getRemember()) ? auth.getRememberDays() : 0; + UserEntity user = new UserEntity(); + user.setUsername(username); + user.setPassword(body.getPassword()); + return userService.authenticate(user, TimeUnit.DAYS.toMillis(days)); + } + + @Override + public CommonResponse signup(AuthBody body) + { + AuthSettings auth = settings.auth(); + if (!auth.isRegistration()) { + return CommonResponse.failure("Registration is disabled"); + } + CommonResponse failed = captcha(auth, body); + if (failed != null) { + return failed; + } + if (body.getUsername() == null || !USERNAME.matcher(body.getUsername()).matches()) { + return CommonResponse.failure("The username has 3 to 32 letters, digits or underscores"); + } + UserEntity user = new UserEntity(); + if (verification(auth, settings.mail())) { + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String email = normalize(body.getEmail()); + if (userRepository.findByEmail(email).isPresent()) { + return CommonResponse.failure("The email is already registered"); + } + if (!codes.verify(email, EmailCodes.Purpose.SIGNUP, body.getCode())) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + user.setEmail(email); + } + user.setUsername(body.getUsername()); + user.setPassword(body.getPassword()); + return userService.saveOrUpdate(user); + } + + @Override + public CommonResponse sendCode(EmailCodeBody body) + { + AuthSettings auth = settings.auth(); + MailSettings mail = settings.mail(); + EmailCodes.Purpose purpose; + try { + purpose = EmailCodes.Purpose.valueOf(StringUtils.defaultString(body.getPurpose()).toUpperCase(Locale.ROOT)); + } + catch (IllegalArgumentException e) { + return CommonResponse.failure("Unknown code purpose"); + } + boolean enabled = purpose == EmailCodes.Purpose.SIGNUP ? verification(auth, mail) : reset(auth, mail); + if (!enabled) { + return CommonResponse.failure("This feature is not enabled"); + } + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String email = normalize(body.getEmail()); + long waiting = codes.waiting(email, purpose); + if (waiting > 0) { + return CommonResponse.failure("Wait " + waiting + " seconds before asking for another code"); + } + boolean registered = userRepository.findByEmail(email).isPresent(); + if (purpose == EmailCodes.Purpose.SIGNUP && registered) { + return CommonResponse.failure("The email is already registered"); + } + String code = codes.issue(email, purpose); + // A reset for an unknown address answers the same way, so addresses can not be probed. + if (purpose == EmailCodes.Purpose.RESET && !registered) { + return CommonResponse.success(EmailCodes.RESEND_SECONDS); + } + try { + mailService.send(mail, email, "MetaForge 验证码 / Verification code", String.format( + "你的 MetaForge 验证码是 %s,%d 分钟内有效。如非本人操作请忽略本邮件。%nYour MetaForge verification code is %s. It expires in %d minutes; ignore this mail if you did not ask for it.", + code, EmailCodes.MINUTES, code, EmailCodes.MINUTES)); + } + catch (MessagingException | RuntimeException e) { + codes.discard(email, purpose); + log.warn("Sending a code to {} failed", email, e); + return CommonResponse.failure("Failed to send mail: " + e.getMessage()); + } + return CommonResponse.success(EmailCodes.RESEND_SECONDS); + } + + @Override + public CommonResponse reset(PasswordResetBody body) + { + if (!reset(settings.auth(), settings.mail())) { + return CommonResponse.failure("This feature is not enabled"); + } + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String password = body.getPassword(); + if (password == null || password.length() < PASSWORD_MIN || password.length() > PASSWORD_MAX) { + return CommonResponse.failure("The password has 6 to 120 characters"); + } + String email = normalize(body.getEmail()); + if (!codes.verify(email, EmailCodes.Purpose.RESET, body.getCode())) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + Optional user = userRepository.findByEmail(email); + if (user.isEmpty()) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + UserEntity value = user.get(); + value.setPassword(encoder.encode(password)); + userRepository.save(value); + return CommonResponse.success(Boolean.TRUE); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ConfigureServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ConfigureServiceImpl.java deleted file mode 100644 index 61ff958450..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ConfigureServiceImpl.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.devlive.metaforge.service.service.impl; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import lombok.extern.slf4j.Slf4j; -import org.devlive.metaforge.common.response.CommonResponse; -import org.devlive.metaforge.service.itransient.configuration.CategoryConfiguration; -import org.devlive.metaforge.service.itransient.configuration.Configuration; -import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; -import org.devlive.metaforge.service.service.ConfigureService; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Objects.requireNonNull; - -@Service -@Slf4j -public class ConfigureServiceImpl - implements ConfigureService -{ - private final Environment environment; - - public ConfigureServiceImpl(Environment environment) - { - this.environment = environment; - } - - @Override - public CommonResponse getExecutor() - { - try { - Configuration config = new Configuration(); - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - Path baseDir = Paths.get(Path.of(requireNonNull(environment.getProperty("spring.config.location")), "executor").toUri()); - - Path categoryPath = baseDir.resolve("category.yaml"); - if (Files.exists(categoryPath)) { - List categories = mapper.readValue( - categoryPath.toFile(), - mapper.getTypeFactory().constructCollectionType(List.class, CategoryConfiguration.class) - ); - config.setCategories(categories); - } - - try (Stream files = Files.list(baseDir)) { - List nodes = files - .filter(path -> path.toString().endsWith(".yaml")) - .filter(path -> !path.getFileName().toString().equals("category.yaml")) - .map(path -> { - try { - return mapper.readValue(path.toFile(), NodeConfiguration.class); - } - catch (IOException e) { - log.debug("Failed to read node configuration", e); - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - config.setNodes(nodes); - } - - return CommonResponse.success(config); - } - catch (IOException e) { - log.warn("Failed to get executor configuration", e); - return CommonResponse.failure("Failed to get executor configuration"); - } - } -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DashboardServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DashboardServiceImpl.java index b1c44e1d1a..21f10df0b0 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DashboardServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DashboardServiceImpl.java @@ -1,31 +1,73 @@ package org.devlive.metaforge.service.service.impl; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.StringUtils; import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.DashboardShareBody; +import org.devlive.metaforge.service.dashboard.share.DashboardViewerDocument; import org.devlive.metaforge.service.entity.DashboardEntity; import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.enums.SharePermission; +import org.devlive.metaforge.service.event.NotificationEventPublisher; +import org.devlive.metaforge.service.persistence.Persistence; import org.devlive.metaforge.service.repository.BaseRepository; import org.devlive.metaforge.service.repository.DashboardRepository; import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.RoleRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.repository.jdbc.DashboardShareRepository; import org.devlive.metaforge.service.repository.jdbc.LinkRepository; +import org.devlive.metaforge.service.security.RunAs; import org.devlive.metaforge.service.security.UserDetailsService; import org.devlive.metaforge.service.service.DashboardService; +import org.devlive.metaforge.service.service.ReportService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.security.SecureRandom; import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.Date; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +/** + * Data screens with sharing. The owner shares a screen with users, roles or every signed-in user, + * each with VIEW, EDIT or PUBLISH, and chooses which widgets viewers see. Viewers receive only the + * published design and the reports their widgets draw; editors work on the draft; only the owner + * deletes a screen or changes its sharing. Reports always run as the owner, who exposes them. + */ @Service @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") public class DashboardServiceImpl implements DashboardService { + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final int CANDIDATES = 20; + private static final java.util.regex.Pattern PUBLIC_TOKEN = java.util.regex.Pattern.compile("^[A-Za-z0-9_-]{32}$"); private final DashboardRepository repository; private final ReportRepository reportRepository; private final LinkRepository linkRepository; + private final SecureRandom random = new SecureRandom(); public DashboardServiceImpl(DashboardRepository repository, ReportRepository reportRepository, LinkRepository linkRepository) { @@ -34,59 +76,629 @@ public DashboardServiceImpl(DashboardRepository repository, ReportRepository rep this.linkRepository = linkRepository; } + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + private UserRepository users; + private RoleRepository roles; + private DashboardShareRepository shares; + private ReportService reports; + + @Autowired + public void setUsers(UserRepository users) + { + this.users = users; + } + + @Autowired + public void setRoles(RoleRepository roles) + { + this.roles = roles; + } + + @Autowired + public void setShares(DashboardShareRepository shares) + { + this.shares = shares; + } + + @Autowired + public void setReports(@Lazy ReportService reports) + { + this.reports = reports; + } + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + @Override @Transactional public CommonResponse saveOrUpdate(BaseRepository baseRepository, DashboardEntity configure) { - configure.setUser(UserDetailsService.getUser()); + var user = UserDetailsService.getUser(); + if (user.getId() == null) { + return CommonResponse.failure("Dashboard not found"); + } + Long ownerId = user.getId(); + Set linked = new HashSet<>(); + if (configure.getId() != null) { + var existing = repository.findById(configure.getId()).orElse(null); + SharePermission permission = existing == null ? null : access(existing, user.getId()); + if (permission == null || !permission.atLeast(SharePermission.EDIT)) { + return CommonResponse.failure("Dashboard not found"); + } + ownerId = existing.getUserId(); + configure.setCreateTime(existing.getCreateTime()); + configure.setActive(existing.isActive()); + if (configure.getAvatar() == null) { + configure.setAvatar(existing.getAvatar()); + } + if (!permission.atLeast(SharePermission.PUBLISH)) { + // Editors save drafts; the published screen stays as the last publisher left it. + Optional kept = keepPublished(configure.getConfigure(), existing.getConfigure()); + if (kept.isEmpty()) { + return CommonResponse.failure("Only users allowed to publish can change the published screen"); + } + configure.setConfigure(kept.get()); + } + linked.addAll(linkRepository.reportIdsOfDashboard(existing.getId())); + } + // Saving never changes the owner, whoever edits the screen. + UserEntity owner = new UserEntity(); + owner.setId(ownerId); + configure.setUser(owner); + configure.setUserId(ownerId); List reportIds = new ArrayList<>(); if (configure.getReports() != null) { - configure.getReports() - .forEach(entity -> reportRepository.findByCode(entity.getCode()) - .ifPresentOrElse( - report -> { - entity.setId(report.getId()); - reportIds.add(report.getId()); - }, - () -> { - throw new RuntimeException("Report [ " + entity.getCode() + " ] not found"); - } - )); + for (ReportEntity requested : configure.getReports()) { + Long screenOwner = ownerId; + var report = reportRepository.findByCode(requested.getCode()) + .filter(value -> Objects.equals(value.getUserId(), screenOwner) || linked.contains(value.getId())) + .orElse(null); + if (report == null) { + return CommonResponse.failure("Dashboard report not found"); + } + reportIds.add(report.getId()); + } } + boolean created = configure.getId() == null; DashboardEntity saved = repository.save(configure); linkRepository.replaceDashboardReports(saved.getId(), reportIds); + if (created) { + notifications.publish(NotificationScene.DASHBOARD_CREATED, String.format("仪表盘「%s」已创建", saved.getName()), + String.format("仪表盘「%s」已创建,包含 %d 个报表。", saved.getName(), reportIds.size()), EntityType.DASHBOARD, saved.getCode(), saved.getName()); + } + saved.setPermission((created ? SharePermission.OWNER : access(saved, user.getId())).name()); return CommonResponse.success(saved); } @Override public CommonResponse getById(BaseRepository baseRepository, Long id) { - CommonResponse response = DashboardService.super.getById(baseRepository, id); - return withReports(response); + return load(repository.findById(id)); } @Override public CommonResponse getByCode(BaseRepository baseRepository, String code) { - CommonResponse response = DashboardService.super.getByCode(baseRepository, code); - return withReports(response); + return load(repository.findByCode(code)); + } + + /** A screen as the signed-in user may see it: whole for editors, the viewer document for viewers. */ + private CommonResponse load(Optional match) + { + Long userId = UserDetailsService.getUser().getId(); + DashboardEntity dashboard = match.orElse(null); + SharePermission permission = dashboard == null ? null : access(dashboard, userId); + if (permission == null) { + return CommonResponse.failure("Dashboard not found"); + } + Persistence.references().load(dashboard); + dashboard.setPermission(permission.name()); + if (permission != SharePermission.OWNER) { + dashboard.setOwnerName(ownerName(dashboard.getUserId())); + } + if (permission == SharePermission.VIEW) { + Optional viewer = DashboardViewerDocument.of(dashboard.getConfigure(), visibleWidgets(dashboard.getId())); + dashboard.setConfigure(viewer.orElse(null)); + Set codes = viewer.map(DashboardViewerDocument::reportCodes).orElse(Set.of()); + return withReports(CommonResponse.success(dashboard), codes); + } + return withReports(CommonResponse.success(dashboard), null); } @Override @Transactional public CommonResponse deleteByCode(BaseRepository baseRepository, String code) { - repository.findByCode(code).ifPresent(dashboard -> linkRepository.deleteDashboard(dashboard.getId())); - return DashboardService.super.deleteByCode(baseRepository, code); + Long userId = UserDetailsService.getUser().getId(); + DashboardEntity dashboard = owned(code, userId).orElse(null); + if (dashboard == null) { + return CommonResponse.failure("Dashboard not found"); + } + linkRepository.deleteDashboard(dashboard.getId()); + if (shares != null) { + shares.delete(dashboard.getId()); + } + repository.delete(dashboard); + if (users != null) { + users.clearHomePage(code); + } + return CommonResponse.success(code); + } + + @Override + @Transactional + public CommonResponse setHomePage(String code) + { + Long userId = UserDetailsService.getUser().getId(); + if (userId == null || users == null) { + return CommonResponse.failure("Dashboard not found"); + } + if (code != null) { + DashboardEntity dashboard = repository.findByCode(code).filter(value -> access(value, userId) != null).orElse(null); + if (dashboard == null) { + return CommonResponse.failure("Dashboard not found"); + } + if (!published(dashboard.getConfigure())) { + return CommonResponse.failure("Publish the data screen before using it as the home page"); + } + } + users.updateHomePage(userId, code); + return CommonResponse.success(code); + } + + @Override + public CommonResponse> shared() + { + Long userId = UserDetailsService.getUser().getId(); + if (userId == null || shares == null) { + return CommonResponse.success(List.of()); + } + Map granted = shares.sharedWith(userId, linkRepository.roleIdsOfUser(userId)); + List result = StreamSupport.stream(repository.findAllById(granted.keySet()).spliterator(), false) + .filter(dashboard -> !Objects.equals(dashboard.getUserId(), userId)) + .peek(dashboard -> { + SharePermission permission = granted.get(dashboard.getId()); + dashboard.setPermission(permission.name()); + dashboard.setOwnerName(ownerName(dashboard.getUserId())); + if (permission == SharePermission.VIEW) { + dashboard.setConfigure(DashboardViewerDocument.of(dashboard.getConfigure(), visibleWidgets(dashboard.getId())).orElse(null)); + } + }) + .sorted(Comparator.comparing((DashboardEntity dashboard) -> dashboard.getUpdateTime() == null ? new Date(0) : dashboard.getUpdateTime()).reversed()) + .collect(Collectors.toList()); + return CommonResponse.success(result); + } + + @Override + public CommonResponse> shared(String keyword, String sort) + { + if (sort == null || !List.of("updated", "created", "name").contains(sort) || (keyword != null && keyword.length() > 200)) { + return CommonResponse.failure("Unsupported screen search"); + } + CommonResponse> response = shared(); + if (!Boolean.TRUE.equals(response.getStatus())) { + return response; + } + String query = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT); + Comparator order = "name".equals(sort) + ? Comparator.comparing(item -> item.getName() == null ? "" : item.getName(), String.CASE_INSENSITIVE_ORDER) + : Comparator.comparing((DashboardEntity item) -> { + Date date = "created".equals(sort) ? item.getCreateTime() : item.getUpdateTime(); + return date == null ? new Date(0) : date; + }).reversed(); + // Search only the already-authorized result, including the viewer's widget restrictions. + return CommonResponse.success(response.getData().stream() + .filter(item -> query.isEmpty() || (item.getName() != null && item.getName().toLowerCase(java.util.Locale.ROOT).contains(query))) + .sorted(order).collect(Collectors.toList())); + } + + @Override + public CommonResponse>> refreshChart(String code, String reportCode) + { + Long userId = UserDetailsService.getUser().getId(); + DashboardEntity dashboard = repository.findByCode(code).orElse(null); + SharePermission permission = dashboard == null ? null : access(dashboard, userId); + if (permission == null || reports == null) { + return CommonResponse.failure("Dashboard not found"); + } + ReportEntity report = reportRepository.findByCode(reportCode) + .filter(value -> Objects.equals(value.getUserId(), dashboard.getUserId())) + .filter(value -> linkRepository.reportIdsOfDashboard(dashboard.getId()).contains(value.getId())) + .orElse(null); + if (report == null) { + return CommonResponse.failure("Dashboard report not found"); + } + if (permission == SharePermission.VIEW) { + Set visible = DashboardViewerDocument.of(dashboard.getConfigure(), visibleWidgets(dashboard.getId())) + .map(DashboardViewerDocument::reportCodes).orElse(Set.of()); + if (!visible.contains(reportCode)) { + return CommonResponse.failure("Dashboard report not found"); + } + } + if (permission == SharePermission.OWNER) { + return reports.refreshChart(reportCode); + } + // The owner shares the screen and with it the right to read its reports; they run as the owner. + UserEntity owner = users == null ? null : users.findById(dashboard.getUserId()).orElse(null); + return RunAs.user(owner, () -> reports.refreshChart(reportCode)); + } + + @Override + public CommonResponse getShare(String code) + { + DashboardEntity dashboard = owned(code, UserDetailsService.getUser().getId()).orElse(null); + if (dashboard == null || shares == null) { + return CommonResponse.failure("Dashboard not found"); + } + return CommonResponse.success(shareOf(dashboard.getId())); + } + + @Override + @Transactional + public CommonResponse saveShare(String code, DashboardShareBody body) + { + Long userId = UserDetailsService.getUser().getId(); + DashboardEntity dashboard = owned(code, userId).orElse(null); + if (dashboard == null || shares == null || body == null) { + return CommonResponse.failure("Dashboard not found"); + } + String everyone = null; + if (StringUtils.isNotBlank(body.getEveryone())) { + Optional permission = SharePermission.grantable(body.getEveryone()); + if (permission.isEmpty()) { + return CommonResponse.failure("Unknown permission " + body.getEveryone()); + } + everyone = permission.get().name(); + } + List members = new ArrayList<>(); + Set seen = new HashSet<>(); + for (DashboardShareBody.Member member : body.getMembers() == null ? List.of() : body.getMembers()) { + String type = StringUtils.upperCase(StringUtils.trimToEmpty(member.getType()), Locale.ROOT); + Optional permission = SharePermission.grantable(member.getPermission()); + boolean known = DashboardShareRepository.USER.equals(type) ? member.getId() != null && !member.getId().equals(userId) && users.existsById(member.getId()) + : DashboardShareRepository.ROLE.equals(type) && member.getId() != null && roles.existsById(member.getId()); + if (!known || permission.isEmpty()) { + return CommonResponse.failure("Share with existing users or roles, each with VIEW, EDIT or PUBLISH"); + } + if (seen.add(type + ":" + member.getId())) { + members.add(new DashboardShareRepository.Member(type, member.getId(), permission.get().name())); + } + } + String widgets = null; + if (body.getWidgets() != null) { + Set available = DashboardViewerDocument.publishedWidgetIds(dashboard.getConfigure()); + List chosen = body.getWidgets().stream().filter(available::contains).distinct().collect(Collectors.toList()); + if (chosen.isEmpty()) { + return CommonResponse.failure("Choose at least one published widget to share"); + } + try { + widgets = MAPPER.writeValueAsString(chosen); + } + catch (JsonProcessingException e) { + return CommonResponse.failure("Choose at least one published widget to share"); + } + } + DashboardShareRepository.Share share = shares.find(dashboard.getId()).orElseGet(DashboardShareRepository.Share::new); + share.setDashboardId(dashboard.getId()); + DashboardShareBody.PublicLink link = body.getPublicLink() == null ? new DashboardShareBody.PublicLink() : body.getPublicLink(); + if (link.isEnabled()) { + if (!published(dashboard.getConfigure())) { + return CommonResponse.failure("Publish the data screen before sharing a public link"); + } + if (link.getExpiresAt() != null && link.getExpiresAt() <= System.currentTimeMillis()) { + return CommonResponse.failure("The expiry time must be in the future"); + } + } + Optional origins = ReportShareServiceImpl.normalizeOrigins(link.getOrigins()); + if (origins.isEmpty()) { + return CommonResponse.failure("Allowed sites must be origins such as https://example.com, at most " + ReportShareServiceImpl.MAX_ORIGINS); + } + if (link.isRotate() || share.getPublicToken() == null) { + share.setPublicToken(newToken()); + } + share.setEveryonePermission(everyone); + share.setWidgets(widgets); + share.setPublicEnabled(link.isEnabled()); + share.setPublicEmbed(link.isEmbed()); + share.setPublicOrigins(origins.get().isEmpty() ? null : origins.get()); + share.setPublicExpiresAt(link.getExpiresAt() == null ? null : new Date(link.getExpiresAt())); + shares.save(share); + shares.replaceMembers(dashboard.getId(), members); + return CommonResponse.success(shareOf(dashboard.getId())); + } + + @Override + public CommonResponse> shareCandidates(String keyword) + { + Long userId = UserDetailsService.getUser().getId(); + String text = StringUtils.lowerCase(StringUtils.trimToEmpty(keyword), Locale.ROOT); + List result = new ArrayList<>(); + StreamSupport.stream(users.findAll().spliterator(), false) + .filter(user -> !Objects.equals(user.getId(), userId) && user.isActive()) + .filter(user -> StringUtils.lowerCase(StringUtils.defaultString(user.getUsername()), Locale.ROOT).contains(text)) + .limit(CANDIDATES) + .forEach(user -> result.add(new DashboardShareBody.Member(DashboardShareRepository.USER, user.getId(), user.getUsername(), null))); + StreamSupport.stream(roles.findAll().spliterator(), false) + .filter(role -> StringUtils.lowerCase(StringUtils.defaultString(role.getName()), Locale.ROOT).contains(text)) + .limit(CANDIDATES) + .forEach(role -> result.add(new DashboardShareBody.Member(DashboardShareRepository.ROLE, role.getId(), role.getName(), null))); + return CommonResponse.success(result); + } + + @Override + public CommonResponse> publicView(String token) + { + return publicly(() -> { + Optional shared = publicScreen(token); + if (shared.isEmpty()) { + return CommonResponse.failure("Dashboard not found"); + } + Set codes = DashboardViewerDocument.reportCodes(shared.get().document); + List> visible = new ArrayList<>(); + reportRepository.findAllById(linkRepository.reportIdsOfDashboard(shared.get().dashboard.getId())).forEach(report -> { + if (Objects.equals(report.getUserId(), shared.get().dashboard.getUserId()) && codes.contains(report.getCode())) { + // Anonymous visitors get what the charts draw: never SQL, data sources or the owner. + Map item = new java.util.LinkedHashMap<>(); + item.put("code", report.getCode()); + item.put("name", report.getName()); + item.put("type", report.getType() == null ? null : report.getType().name()); + item.put("realtime", report.isRealtime()); + item.put("configure", report.getConfigure()); + visible.add(item); + } + }); + Map view = new java.util.LinkedHashMap<>(); + view.put("name", shared.get().dashboard.getName()); + view.put("configure", shared.get().document); + view.put("reports", visible); + view.put("embed", shared.get().share.isPublicEmbed()); + return CommonResponse.success(view); + }); + } + + @Override + public CommonResponse>> publicRefresh(String token, String reportCode) + { + return publicly(() -> { + Optional shared = publicScreen(token); + if (shared.isEmpty() || reports == null || !DashboardViewerDocument.reportCodes(shared.get().document).contains(reportCode)) { + return CommonResponse.failure("Dashboard report not found"); + } + DashboardEntity dashboard = shared.get().dashboard; + boolean linked = reportRepository.findByCode(reportCode) + .filter(value -> Objects.equals(value.getUserId(), dashboard.getUserId())) + .filter(value -> linkRepository.reportIdsOfDashboard(dashboard.getId()).contains(value.getId())) + .isPresent(); + if (!linked) { + return CommonResponse.failure("Dashboard report not found"); + } + UserEntity owner = users == null ? null : users.findById(dashboard.getUserId()).orElse(null); + return RunAs.user(owner, () -> reports.refreshChart(reportCode)); + }); + } + + @Override + public Optional publicFrameAncestors(String token) + { + try { + return publicScreen(token) + .filter(shared -> shared.share.isPublicEmbed()) + .map(shared -> StringUtils.isBlank(shared.share.getPublicOrigins()) ? "*" + : "'self' " + String.join(" ", shared.share.getPublicOrigins().split("\n"))); + } + catch (RuntimeException e) { + return Optional.empty(); + } + } + + /** A screen behind a public link, with the document anonymous visitors see. */ + private static final class Shared + { + private final DashboardEntity dashboard; + private final DashboardShareRepository.Share share; + private final String document; + + private Shared(DashboardEntity dashboard, DashboardShareRepository.Share share, String document) + { + this.dashboard = dashboard; + this.share = share; + this.document = document; + } + } + + private Optional publicScreen(String token) + { + if (shares == null || token == null || !PUBLIC_TOKEN.matcher(token).matches()) { + return Optional.empty(); + } + return shares.findPublic(token) + .flatMap(id -> shares.find(id).filter(share -> share.isPublicEnabled() && token.equals(share.getPublicToken())) + .filter(share -> share.getPublicExpiresAt() == null || share.getPublicExpiresAt().getTime() > System.currentTimeMillis()) + .flatMap(share -> repository.findById(id).filter(DashboardEntity::isActive) + .flatMap(dashboard -> DashboardViewerDocument.of(dashboard.getConfigure(), visibleWidgets(id)) + .map(document -> new Shared(dashboard, share, document))))); + } + + /** Anonymous reads report a generic failure instead of internal errors such as SQL text. */ + private static CommonResponse publicly(java.util.function.Supplier> action) + { + try { + return action.get(); + } + catch (RuntimeException e) { + return CommonResponse.failure("Dashboard data is temporarily unavailable"); + } + } + + /** + * What the user may do with a screen: OWNER for its own user, otherwise the highest of a direct + * grant, a grant to one of the user's roles and the grant to everyone; null when not shared. + */ + SharePermission access(DashboardEntity dashboard, Long userId) + { + if (userId == null || dashboard == null) { + return null; + } + if (Objects.equals(dashboard.getUserId(), userId)) { + return SharePermission.OWNER; + } + if (shares == null || dashboard.getId() == null) { + return null; + } + SharePermission best = shares.find(dashboard.getId()).flatMap(share -> SharePermission.grantable(share.getEveryonePermission())).orElse(null); + List roleIds = linkRepository.roleIdsOfUser(userId); + for (DashboardShareRepository.Member member : shares.members(dashboard.getId())) { + boolean mine = DashboardShareRepository.USER.equals(member.getType()) && Objects.equals(member.getId(), userId) + || DashboardShareRepository.ROLE.equals(member.getType()) && roleIds.contains(member.getId()); + SharePermission granted = mine ? SharePermission.grantable(member.getPermission()).orElse(null) : null; + if (granted != null && (best == null || granted.atLeast(best))) { + best = granted; + } + } + return best; + } + + private Optional owned(String code, Long userId) + { + return userId == null ? Optional.empty() : repository.findByCode(code).filter(value -> Objects.equals(value.getUserId(), userId)); + } + + /** Widget ids viewers see, or null for the whole screen. */ + private Set visibleWidgets(Long dashboardId) + { + String widgets = shares == null ? null : shares.find(dashboardId).map(DashboardShareRepository.Share::getWidgets).orElse(null); + if (widgets == null) { + return null; + } + try { + return new LinkedHashSet<>(MAPPER.readValue(widgets, new TypeReference>() {})); + } + catch (JsonProcessingException e) { + // An unreadable selection shows nothing rather than everything. + return Set.of(); + } + } + + private DashboardShareBody shareOf(Long dashboardId) + { + DashboardShareBody body = new DashboardShareBody(); + Optional share = shares.find(dashboardId); + body.setEveryone(share.map(DashboardShareRepository.Share::getEveryonePermission).orElse(null)); + Set widgets = share.isPresent() ? visibleWidgets(dashboardId) : null; + body.setWidgets(widgets == null ? null : new ArrayList<>(widgets)); + List members = new ArrayList<>(); + for (DashboardShareRepository.Member member : shares.members(dashboardId)) { + String name = DashboardShareRepository.USER.equals(member.getType()) + ? users.findById(member.getId()).map(UserEntity::getUsername).orElse(null) + : roles.findById(member.getId()).map(role -> role.getName()).orElse(null); + if (name != null) { + members.add(new DashboardShareBody.Member(member.getType(), member.getId(), name, member.getPermission())); + } + } + body.setMembers(members); + DashboardShareBody.PublicLink link = new DashboardShareBody.PublicLink(); + share.ifPresent(value -> { + link.setEnabled(value.isPublicEnabled()); + link.setEmbed(value.isPublicEmbed()); + link.setOrigins(StringUtils.defaultString(value.getPublicOrigins())); + link.setExpiresAt(value.getPublicExpiresAt() == null ? null : value.getPublicExpiresAt().getTime()); + link.setToken(value.getPublicToken()); + }); + body.setPublicLink(link); + return body; + } + + private String ownerName(Long ownerId) + { + return users == null || ownerId == null ? null : users.findById(ownerId).map(UserEntity::getUsername).orElse(null); + } + + private String newToken() + { + byte[] bytes = new byte[24]; + random.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + /** + * The incoming document with the stored published design kept, for editors without the right to + * publish. Empty when the stored screen predates drafts and cannot be kept apart. + */ + static Optional keepPublished(String incoming, String stored) + { + try { + JsonNode next = incoming == null ? null : MAPPER.readTree(incoming); + JsonNode previous = stored == null || stored.isBlank() ? null : MAPPER.readTree(stored); + if (previous != null && !previous.isNull() && !(previous.isObject() && previous.path("version").asInt() == 3)) { + return Optional.empty(); + } + if (next == null || !next.isObject()) { + return Optional.empty(); + } + ObjectNode document = (ObjectNode) next; + JsonNode published = previous == null ? null : previous.get("published"); + if (published == null || published.isNull()) { + document.remove("published"); + } + else { + document.set("published", published); + } + return Optional.of(MAPPER.writeValueAsString(document)); + } + catch (JsonProcessingException e) { + return Optional.empty(); + } + } + + /** + * Whether a saved document has a published version, read the way the editor reads it: a + * version 3 document carries {@code published}; older layouts count as published once saved. + */ + public static boolean published(String configure) + { + if (configure == null || configure.isBlank()) { + return false; + } + try { + JsonNode root = MAPPER.readTree(configure); + if (root != null && root.isObject() && root.path("version").asInt() == 3) { + return root.path("published").isObject(); + } + return root != null && !root.isNull() && !root.isMissingNode(); + } + catch (JsonProcessingException e) { + return false; + } } - private CommonResponse withReports(CommonResponse response) + /** Links the screen's reports owned by its owner; with {@code only} set, just those codes. */ + private CommonResponse withReports(CommonResponse response, Set only) { DashboardEntity dashboard = response.getData(); if (response.getStatus() && dashboard != null) { - Set reports = new HashSet<>(); - reportRepository.findAllById(linkRepository.reportIdsOfDashboard(dashboard.getId())).forEach(reports::add); - dashboard.setReports(reports); + Set linked = new HashSet<>(); + reportRepository.findAllById(linkRepository.reportIdsOfDashboard(dashboard.getId())).forEach(report -> { + if (Objects.equals(report.getUserId(), dashboard.getUserId()) && (only == null || only.contains(report.getCode()))) { + linked.add(report); + } + }); + Persistence.references().load(linked); + if (!SharePermission.OWNER.name().equals(dashboard.getPermission())) { + // Members see what the charts draw, never the owner's SQL, data sources or report links. + linked.forEach(report -> { + report.setQuery(null); + report.setSource(null); + report.setSourceId(null); + report.setUser(null); + report.setShareToken(null); + report.setShareEnabled(false); + report.setShareOrigins(null); + }); + } + dashboard.setReports(linked); } return response; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DataSetServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DataSetServiceImpl.java index 8cb66959db..b282e9c225 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DataSetServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DataSetServiceImpl.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.devlive.metaforge.common.enums.DataSetState; +import org.devlive.metaforge.common.enums.ServiceState; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.common.sql.SqlBuilder; import org.devlive.metaforge.common.sql.configure.SqlBody; @@ -18,6 +19,7 @@ import org.devlive.metaforge.executor.ExecutorService; import org.devlive.metaforge.executor.common.RunProtocol; import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.common.RunStateMachine; import org.devlive.metaforge.executor.configure.ExecutorConfigure; import org.devlive.metaforge.executor.configure.ExecutorRequest; import org.devlive.metaforge.executor.configure.ExecutorResponse; @@ -29,13 +31,16 @@ import org.devlive.metaforge.scheduler.SchedulerRequest; import org.devlive.metaforge.scheduler.SchedulerService; import org.devlive.metaforge.service.adapter.PageRequestAdapter; -import org.devlive.metaforge.service.annotation.SendNotification; import org.devlive.metaforge.service.body.FilterBody; import org.devlive.metaforge.service.body.PipelineFieldBody; import org.devlive.metaforge.service.body.adhoc.Adhoc; import org.devlive.metaforge.service.common.ConfigureUtils; import org.devlive.metaforge.service.common.FolderUtils; import org.devlive.metaforge.service.dataset.DatasetVirtualColumns; +import org.devlive.metaforge.service.dataset.sync.SyncActiveView; +import org.devlive.metaforge.service.dataset.sync.SyncJob; +import org.devlive.metaforge.service.dataset.sync.SyncJobManager; +import org.devlive.metaforge.service.dataset.sync.SyncMetricsView; import org.devlive.metaforge.service.entity.DataSetColumnEntity; import org.devlive.metaforge.service.entity.DataSetEntity; import org.devlive.metaforge.service.entity.DatasetHistoryEntity; @@ -46,7 +51,7 @@ import org.devlive.metaforge.service.enums.ColumnType; import org.devlive.metaforge.service.enums.CreatedMode; import org.devlive.metaforge.service.enums.EntityType; -import org.devlive.metaforge.service.enums.NotificationType; +import org.devlive.metaforge.service.enums.NotificationScene; import org.devlive.metaforge.service.enums.QueryMode; import org.devlive.metaforge.service.enums.SyncMode; import org.devlive.metaforge.service.event.NotificationEventPublisher; @@ -101,16 +106,11 @@ public class DataSetServiceImpl implements DataSetService { - // 同步期间每 SIZE_REFRESH_EVERY 次进度回调才刷一次 totalSize(查 ClickHouse system.parts 较重) + // 同步期间每 SIZE_REFRESH_EVERY 次指标落库才刷一次 totalSize(查 ClickHouse system.parts 较重) private static final int SIZE_REFRESH_EVERY = 10; - // 已请求停止但还没真正停下来的 history id 集合:进度回调据此把 state 维持为 STOPPING 而不是 RUNNING - // Set of history ids whose sync has been asked to stop but hasn't reached its cancellation checkpoint yet. - private static final java.util.Set stoppingHistoryIds = java.util.concurrent.ConcurrentHashMap.newKeySet(); - - // 本次同步的 tunable 字段临时覆盖。由 syncData(code, overrides) 在异步线程入口写入,由内部 sync 体读取后清理。 - // Per-invocation tunable overrides; set by the public syncData entry, read inside the async body. - private static final ThreadLocal> currentSyncOverrides = new ThreadLocal<>(); + // 同步完成后刷新表元数据(行数、大小)的服务端阶段 / service-side stage that refreshes table metadata after a sync + private static final String SYNC_STAGE_FINALIZE = "FINALIZE"; public final DataSetColumnRepository columnRepository; private final DataSetRepository repository; @@ -123,8 +123,9 @@ public class DataSetServiceImpl private final NotificationRepository notificationRepository; private final NotificationEventPublisher eventPublisher; private final RuntimeConfigureService runtimeConfigureService; + private final SyncJobManager syncJobManager; - public DataSetServiceImpl(DataSetRepository repository, DataSetColumnRepository columnRepository, DatasetHistoryRepository historyRepository, PluginManager pluginManager, InitializerConfigure initializerConfigure, org.quartz.Scheduler scheduler, Environment environment, SourceRepository sourceRepository, NotificationRepository notificationRepository, NotificationEventPublisher eventPublisher, RuntimeConfigureService runtimeConfigureService) + public DataSetServiceImpl(DataSetRepository repository, DataSetColumnRepository columnRepository, DatasetHistoryRepository historyRepository, PluginManager pluginManager, InitializerConfigure initializerConfigure, org.quartz.Scheduler scheduler, Environment environment, SourceRepository sourceRepository, NotificationRepository notificationRepository, NotificationEventPublisher eventPublisher, RuntimeConfigureService runtimeConfigureService, SyncJobManager syncJobManager) { this.repository = repository; this.columnRepository = columnRepository; @@ -137,11 +138,21 @@ public DataSetServiceImpl(DataSetRepository repository, DataSetColumnRepository this.notificationRepository = notificationRepository; this.eventPublisher = eventPublisher; this.runtimeConfigureService = runtimeConfigureService; + this.syncJobManager = syncJobManager; + } + + @Override + public CommonResponse count() + { + Long userId = UserDetailsService.getUser().getId(); + if (userId == null) { + return CommonResponse.failure(ServiceState.USER_UNAUTHORIZED); + } + return CommonResponse.success(repository.countByUserId(userId)); } @Transactional @Override - @SendNotification(type = NotificationType.DYNAMIC) public CommonResponse saveOrUpdate(BaseRepository repository, DataSetEntity configure) { try { @@ -162,12 +173,18 @@ else if (configure.getColumns().stream().anyMatch(column -> column.getId() != nu return CommonResponse.failure("New datasets cannot reuse persisted column identities"); } UserEntity user = UserDetailsService.getUser(); + // Decided before the build assigns an id, which made every new dataset read as an update. + boolean created = configure.getId() == null; java.util.concurrent.ExecutorService service = Executors.newSingleThreadExecutor(); service.submit(() -> { configure.setUser(user); completeState(configure, DataSetState.METADATA_START); startBuild(configure, true); }); + eventPublisher.publish(created ? NotificationScene.DATASET_CREATED : NotificationScene.DATASET_UPDATED, user, + String.format(created ? "数据集「%s」已创建" : "数据集「%s」已更新", configure.getName()), + String.format(created ? "数据集「%s」已创建,正在构建元数据。" : "数据集「%s」的配置已更新,正在重新构建。", configure.getName()), + EntityType.DATASET, configure.getCode(), configure.getName()); return CommonResponse.success(configure); } @@ -214,6 +231,24 @@ public CommonResponse> getColumnsByCode(String code) .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code))); } + /** 默认分片列:主键优先,其次排序键,取其源列名;都没有返回空串 / primary key first, then order-by key; empty when neither exists */ + private static String defaultSplitColumn(Iterable columns) + { + String orderBy = ""; + for (DataSetColumnEntity column : columns) { + if (column.isVirtualColumn() || StringUtils.isBlank(column.getOriginal())) { + continue; + } + if (column.isPrimaryKey()) { + return column.getOriginal(); + } + if (column.isOrderByKey() && orderBy.isEmpty()) { + orderBy = column.getOriginal(); + } + } + return orderBy; + } + private static int parseIntOrDefault(String raw, int fallback) { if (raw == null || raw.isEmpty()) { @@ -429,12 +464,28 @@ public CommonResponse> getHistoryLog(Long id) public CommonResponse stopHistory(Long id) { log.info("Stop history [ {} ] requested", id); + Optional active = syncJobManager.find(id); + if (active.isEmpty()) { + return historyRepository.findById(id) + .map(history -> CommonResponse.failure(RunStateMachine.isTerminal(history.getState()) + ? String.format("Sync history [ %d ] is not running (current state: %s)", id, history.getState()) + : String.format("Sync history [ %d ] is not running on this node", id))) + .orElseGet(() -> CommonResponse.failure(String.format("Sync history [ %d ] not found", id))); + } + SyncJob job = active.get(); + if (syncJobManager.cancelQueued(job)) { + return CommonResponse.success(Boolean.TRUE); + } + // 正在等待重试:工作线程醒来发现已是终态就结束 / waiting to retry: the worker sees the final state and ends + if (job.getState() == RunState.RETRYING && job.transition(RunState.STOPPED, "Cancelled while waiting to retry")) { + return CommonResponse.success(Boolean.TRUE); + } + if (job.getState() != RunState.RUNNING && job.getState() != RunState.PAUSING) { + return CommonResponse.failure(String.format("Sync history [ %d ] cannot be cancelled (current state: %s)", id, job.getState())); + } return historyRepository.findById(id) .map(history -> { - log.info("Stop history [ {} ] state={} taskName={}", id, history.getState(), history.getTaskName()); - if (history.getState() != RunState.RUNNING && history.getState() != RunState.CREATED) { - return CommonResponse.failure(String.format("Sync history [ %d ] is not running (current state: %s)", id, history.getState())); - } + log.info("Stop history [ {} ] state={} taskName={}", id, job.getState(), history.getTaskName()); String taskName = history.getTaskName(); if (StringUtils.isBlank(taskName)) { return CommonResponse.failure("Sync history has no taskName (older record), cannot stop"); @@ -456,10 +507,8 @@ public CommonResponse stopHistory(Long id) ExecutorResponse stopResponse = executorService.stop(stopRequest); log.info("Stop history [ {} ] executor returned successful={} message={}", id, stopResponse.getSuccessful(), stopResponse.getMessage()); if (Boolean.TRUE.equals(stopResponse.getSuccessful())) { - // 让 UI 立刻看到 STOPPING 中间态;进度回调会继续把它保持住直到真正停下 - stoppingHistoryIds.add(id); - history.setState(RunState.STOPPING); - historyRepository.save(history); + // 让 UI 立刻看到 STOPPING 中间态,直到执行器真正停下 + job.transition(RunState.STOPPING, null); return CommonResponse.success(Boolean.TRUE); } return CommonResponse.failure(stopResponse.getMessage() == null ? "Stop failed" : stopResponse.getMessage()); @@ -467,9 +516,100 @@ public CommonResponse stopHistory(Long id) .orElseGet(() -> CommonResponse.failure(String.format("Sync history [ %d ] not found", id))); } + @Override + public CommonResponse> getActiveSyncs() + { + List views = Lists.newArrayList(); + for (SyncJob job : syncJobManager.active()) { + repository.findById(job.getDatasetId()) + .filter(org.devlive.metaforge.service.service.BaseService::ownedBySelf) + .ifPresent(dataset -> { + org.devlive.metaforge.executor.configure.ExecutorMetrics metrics = job.getMetrics(); + views.add(new SyncActiveView(dataset.getCode(), job.getHistoryId(), job.getState(), + metrics == null ? -1 : metrics.getProgress(), + metrics == null ? 0 : metrics.getRowsWritten(), + metrics == null ? 0 : metrics.getWriteRowsPerSecond())); + }); + } + return CommonResponse.success(views); + } + + @Override + public CommonResponse pauseHistory(Long id) + { + Optional active = syncJobManager.find(id); + if (active.isEmpty() || active.get().getState() != RunState.RUNNING) { + return CommonResponse.failure(String.format("Sync history [ %d ] is not running", id)); + } + SyncJob job = active.get(); + Optional dataset = repository.findById(job.getDatasetId()); + if (dataset.isEmpty()) { + return CommonResponse.failure(String.format("Sync history [ %d ] has no dataset", id)); + } + isSelf(dataset.get()); + String taskName = job.getTaskName(); + if (StringUtils.isBlank(taskName)) { + return CommonResponse.failure("The sync is still starting, try again in a moment"); + } + return pluginManager.getPlugin(dataset.get().getExecutor()) + .map(plugin -> { + ExecutorConfigure placeholder = new ExecutorConfigure(null); + ExecutorResponse response = plugin.getService(ExecutorService.class).pause(new ExecutorRequest(taskName, "", placeholder, placeholder)); + if (!Boolean.TRUE.equals(response.getSuccessful())) { + return CommonResponse.failure(response.getMessage() == null ? "Pause failed" : response.getMessage()); + } + // 暂停可能已经完成,此时这次转换被拒绝,不影响结果 / the pause may already be done; the rejected move changes nothing + job.transition(RunState.PAUSING, null); + return CommonResponse.success(Boolean.TRUE); + }) + .orElseGet(() -> CommonResponse.failure(String.format("Executor [ %s ] not found", dataset.get().getExecutor()))); + } + + @Override + public CommonResponse resumeHistory(Long id) + { + Optional active = syncJobManager.find(id); + if (active.isEmpty() || active.get().getState() != RunState.PAUSED) { + return CommonResponse.failure(String.format("Sync history [ %d ] is not paused", id)); + } + repository.findById(active.get().getDatasetId()).ifPresent(this::isSelf); + return syncJobManager.resume(active.get()) + ? CommonResponse.success(Boolean.TRUE) + : CommonResponse.failure(String.format("Sync history [ %d ] is not paused", id)); + } + + @Override + public CommonResponse retryHistory(Long id) + { + Optional found = historyRepository.findById(id); + if (found.isEmpty()) { + return CommonResponse.failure(String.format("Sync history [ %d ] not found", id)); + } + DatasetHistoryEntity history = found.get(); + if (!SyncJobManager.canRetry(history)) { + return CommonResponse.failure(String.format( + "Sync history [ %d ] cannot run again: it has no checkpoint and running it again could write rows twice. Clear the data and sync again.", id)); + } + Optional dataset = repository.findById(history.getDatasetId()); + if (dataset.isEmpty()) { + return CommonResponse.failure(String.format("Sync history [ %d ] has no dataset", id)); + } + isSelf(dataset.get()); + return syncJobManager.retry(history, syncBody(dataset.get(), UserDetailsService.getUser(), java.util.Collections.emptyMap())) + .map(job -> CommonResponse.success(Boolean.TRUE)) + .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] already has an unfinished sync", dataset.get().getName()))); + } + + @Override + public CommonResponse getHistoryMetrics(Long id) + { + return syncJobManager.view(id) + .map(CommonResponse::success) + .orElseGet(() -> CommonResponse.failure(String.format("Sync history [ %d ] not found", id))); + } + @Override @Transactional - @SendNotification(type = NotificationType.DELETED) public CommonResponse deleteByCode(BaseRepository repository, String code) { return repository.findByCode(code) @@ -493,6 +633,8 @@ public CommonResponse deleteByCode(BaseRepository r historyRepository.deleteAllByDatasetId(value.getId()); repository.delete(value); }); + eventPublisher.publish(NotificationScene.DATASET_DELETED, String.format("数据集「%s」已删除", value.getName()), + String.format("数据集「%s」及其同步数据已删除。", value.getName()), EntityType.DATASET, value.getCode(), value.getName()); return CommonResponse.success(code); }) .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code))); @@ -773,6 +915,53 @@ private void createAndModifyColumn(DataSetEntity entity, Set creat } } + /** + * Records how the latest sync left the dataset's data: a successful sync completes the dataset, + * a failed one marks its data failed and withdraws the completion. A stop the user asked for + * leaves the state as it was. The dataset is re-read so fields saved during the sync survive. + */ + private void recordDataState(Long datasetId, SyncJob job) + { + RunState state = job.getState(); + if (datasetId == null || state != RunState.SUCCESS && state != RunState.FAILURE && state != RunState.TIMEOUT) { + return; + } + List transitions = job.getTransitions(); + String reason = transitions.isEmpty() ? null : transitions.get(transitions.size() - 1).getMessage(); + repository.findById(datasetId).ifPresent(dataset -> { + if (state == RunState.SUCCESS) { + completeState(dataset, DataSetState.DATA_SUCCESS); + completeState(dataset, DataSetState.COMPLETE_SUCCESS); + dataset.setMessage(null); + } + else { + dataset.setState(dataset.getState().stream().filter(item -> !item.name().startsWith("COMPLETE")).collect(Collectors.toList())); + completeState(dataset, DataSetState.DATA_FAILED); + // The dataset's error view shows why the latest sync failed. + dataset.setMessage(reason == null || reason.isBlank() ? state.name() : reason); + } + repository.save(dataset); + }); + } + + /** Tells the user how a sync ended; one they stopped themselves needs no notice. */ + private void notifySynced(DataSetEntity entity, SyncJob job, UserEntity user) + { + RunState state = job.getState(); + if (state != RunState.SUCCESS && state != RunState.FAILURE && state != RunState.TIMEOUT) { + return; + } + boolean succeeded = state == RunState.SUCCESS; + List transitions = job.getTransitions(); + String reason = transitions.isEmpty() ? null : transitions.get(transitions.size() - 1).getMessage(); + String content = succeeded + ? String.format("数据集「%s」同步完成。", entity.getName()) + : String.format("数据集「%s」同步失败:%s", entity.getName(), reason == null || reason.isBlank() ? state.name() : reason); + eventPublisher.publish(succeeded ? NotificationScene.DATASET_SYNC_SUCCEEDED : NotificationScene.DATASET_SYNC_FAILED, user, + String.format(succeeded ? "数据集「%s」同步成功" : "数据集「%s」同步失败", entity.getName()), + content, EntityType.DATASET, entity.getCode(), entity.getName()); + } + private PluginService getOutputPlugin() { return pluginManager.getPlugin(initializerConfigure.getDataSetConfigure().getType()) @@ -781,42 +970,35 @@ private PluginService getOutputPlugin() } @Override - public CommonResponse syncData(String code) + public CommonResponse syncData(String code) { return syncData(code, null); } @Override - public CommonResponse syncData(String code, java.util.Map overrides) + public CommonResponse syncData(String code, java.util.Map overrides) { return repository.findByCode(code) .map(entity -> { UserEntity currentUser = UserDetailsService.getUser(); - java.util.concurrent.ExecutorService service = Executors.newSingleThreadExecutor(); java.util.Map safeOverrides = overrides == null ? java.util.Collections.emptyMap() : new java.util.HashMap<>(overrides); - - service.submit(() -> { - // 在异步线程中设置用户上下文 - setAuthenticationContext(currentUser); - currentSyncOverrides.set(safeOverrides); - try { - DataSetEntity result = syncData(entity, service); - eventPublisher.publishNotificationEvent( - result, - null, - "", - EntityType.DATASET, - NotificationType.SYNCDATA, - new String[] {} - ); - } - finally { - currentSyncOverrides.remove(); - } - }); - return CommonResponse.success(entity); + AtomicReference created = new AtomicReference<>(); + Optional job = syncJobManager.submit(entity.getId(), () -> { + DatasetHistoryEntity history = new DatasetHistoryEntity(); + history.setState(RunState.CREATED); + history.setCreateTime(new Date()); + history.setQuery(entity.getQuery()); + history.setMode(QueryMode.SYNC); + history.setAttempt(1); + history.setDataset(entity); + history.setDatasetId(entity.getId()); + created.set(historyRepository.save(history)); + return created.get(); + }, syncBody(entity, currentUser, safeOverrides)); + return job.map(value -> CommonResponse.success(created.get())) + .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] already has an unfinished sync", entity.getName()))); }) .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code))); } @@ -841,7 +1023,9 @@ public CommonResponse syncBody(DataSetEntity entity, UserEntity user, java.util.Map overrides) + { + return running -> { + // 在工作线程中设置用户上下文,结束后清理,线程会被复用 + setAuthenticationContext(user); + try { + DataSetEntity result = runSync(entity, running, overrides); + // 暂停还没结束,继续后才记录和通知 / a paused sync has not finished, so this waits until it does + if (RunStateMachine.isTerminal(running.getState())) { + recordDataState(result.getId(), running); + notifySynced(result, running, user); + } + } + finally { + SecurityContextHolder.clearContext(); + } + }; + } + /** * Flushes the synchronous data for the given DataSetEntity. * @@ -1111,9 +1315,23 @@ private Column formatColumn(CreatedModel item) return column; } - private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.ExecutorService service) + /** 等到重试时间或任务被取消 / wait for the retry time or a cancel */ + private static void waitForRetry(SyncJob job, int seconds) + { + long deadline = System.currentTimeMillis() + seconds * 1000L; + while (job.getState() == RunState.RETRYING && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(Math.min(200, Math.max(1, deadline - System.currentTimeMillis()))); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private DataSetEntity runSync(DataSetEntity entity, SyncJob job, java.util.Map syncOverrides) { - DatasetHistoryEntity history = new DatasetHistoryEntity(); try { if (entity.getSource() == null || entity.getUser() == null) { Persistence.references().load(entity); @@ -1122,12 +1340,6 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut pluginManager.getPlugin(source.getType()) .ifPresentOrElse(plugin -> { - history.setState(RunState.CREATED); - history.setCreateTime(new Date()); - history.setQuery(entity.getQuery()); - history.setDataset(entity); - historyRepository.save(history); - PluginService inputPlugin = plugin.getService(PluginService.class); pluginManager.getPlugin(entity.getExecutor()) .ifPresentOrElse( @@ -1238,16 +1450,14 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut Sets.newHashSet() ); - String taskName = DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMddHHmmssSSS"); + // 继续或重试沿用首次运行的任务名,日志和坏行文件都在一处 / later runs keep the task name, so logs and bad rows stay together + String taskName = job.getTaskName() != null ? job.getTaskName() : DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMddHHmmssSSS"); String executorKey = executor.getName().toLowerCase(); String workHome = FolderUtils.getWorkHome( initializerConfigure.getDataHome(), entity.getUser().getUsername(), String.join(File.separator, "dataset", "executor", executorKey, taskName) ); - // 把 taskName / workHome 落到 history,供 UI 定位日志文件 - history.setTaskName(taskName); - history.setWorkHome(workHome); // 从 metaforge_configure 取该 executor 的 effective 配置(DB 行覆盖 SPI schema 默认值) java.util.Map executorCfg = runtimeConfigureService.getEffective( @@ -1256,7 +1466,7 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut executor.runtimeConfigures() ); // 本次同步如有用户临时覆盖:只接受 tunable=true 的字段,其余忽略 - java.util.Map overrides = currentSyncOverrides.get(); + java.util.Map overrides = syncOverrides; if (overrides != null && !overrides.isEmpty()) { java.util.Set tunableKeys = executor.runtimeConfigures().stream() .filter(org.devlive.metaforge.plugin.configure.RuntimeConfigureField::isTunable) @@ -1268,15 +1478,37 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut } }); } - // 把本次同步真正生效的配置 JSON 落到 history,便于事后审计 / UI 展示 + // 继续或重试沿用首次运行的配置,断点对应那次的分片 / later runs keep the first run's configuration, which the checkpoint belongs to + if (job.getExecutorConfigure() != null) { + try { + java.util.Map saved = new com.fasterxml.jackson.databind.ObjectMapper().readValue(job.getExecutorConfigure(), + new com.fasterxml.jackson.core.type.TypeReference>() {}); + executorCfg.clear(); + executorCfg.putAll(saved); + } + catch (Exception ex) { + log.warn("Read saved executor configure failed: {}", ex.getMessage()); + } + } + // 未指定分片列时用数据集主键(其次排序键)对应的源列;仅声明了该配置的 executor 需要 + if (executorCfg.containsKey("splitColumn") && StringUtils.isBlank(executorCfg.get("splitColumn"))) { + executorCfg.put("splitColumn", defaultSplitColumn(columnRepository.findAllByDatasetId(entity.getId()))); + } + // taskName / workHome 供 UI 定位日志文件;本次真正生效的配置 JSON 便于事后审计 / UI 展示 + String effectiveConfigure = null; try { - history.setExecutorConfigure(new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(executorCfg)); + effectiveConfigure = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(executorCfg); } catch (Exception ex) { log.warn("Serialize effective executor configure failed: {}", ex.getMessage()); } - historyRepository.save(history); - // timeout 由各 executor 自己的配置决定:LocalExecutor schema 默认 0(不限时), + final String executorConfigure = effectiveConfigure; + job.update(history -> { + history.setTaskName(taskName); + history.setWorkHome(workHome); + history.setExecutorConfigure(executorConfigure); + }); + // timeout 由各 executor 自己的配置决定:MetaForgeExecutor schema 默认 0(不限时), // 未声明该字段的 executor(如 Seatunnel)回退到 600 秒,保持原有行为 int timeout = parseIntOrDefault(executorCfg.get("timeout"), 600); ExecutorRequest request = new ExecutorRequest( @@ -1289,13 +1521,12 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut request.setTimeout(timeout); request.setPluginManager(this.pluginManager); // 把该 executor 的 effective 配置整体作为 options,各执行器各取所需: - // Local 读 fetchSize/batchSize/preCount;Seatunnel 读 home/startScript/way/mode/engine + // MetaForge 读 fetchSize/batchSize/preCount;Seatunnel 读 home/startScript/way/mode/engine request.setOptions(executorCfg); - // 进度回调:每个 batch 写完后更新 metaforge_dataset_history 以及数据集自身的 totalRows / totalSize + // 指标回调:按时间节流更新 metaforge_dataset_history 以及数据集自身的 totalRows / totalSize // totalRows 直接用已写入计数;totalSize 必须查 ClickHouse system.parts,开销大且不可控, // 必须放到独立线程异步执行,否则会阻塞 MySQL 流式 cursor 触发 net_write_timeout - final DatasetHistoryEntity progressHistory = history; final DataSetEntity progressEntity = entity; final java.util.concurrent.atomic.AtomicInteger progressTick = new java.util.concurrent.atomic.AtomicInteger(); final java.util.concurrent.atomic.AtomicBoolean sizeRefreshInFlight = new java.util.concurrent.atomic.AtomicBoolean(false); @@ -1304,36 +1535,19 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut t.setDaemon(true); return t; }); - request.setProgressListener((processed, total) -> { - progressHistory.setProcessedCount(processed); - if (total >= 0) { - progressHistory.setTotalCount(total); - if (total > 0) { - java.math.BigDecimal percent = java.math.BigDecimal - .valueOf(processed * 100.0 / total) - .setScale(2, java.math.RoundingMode.HALF_UP); - progressHistory.setProgress(percent); - } - } - // 收到停止请求后,进度回调不能把 state 写回 RUNNING;保持 STOPPING 直到真正退出 - if (stoppingHistoryIds.contains(progressHistory.getId())) { - progressHistory.setState(RunState.STOPPING); - } - try { - historyRepository.save(progressHistory); - } - catch (Exception ex) { - log.warn("Update sync history progress failed: {}", ex.getMessage()); + request.setMetricsListener(metrics -> { + if (!syncJobManager.onMetrics(job, metrics, false)) { + return; } // 同步期间用已写入行数滚动刷新 dataset.totalRows,方便列表实时看到进度 try { - progressEntity.setTotalRows(processed); + progressEntity.setTotalRows(metrics.getRowsWritten()); repository.save(progressEntity); } catch (Exception ex) { log.warn("Update dataset totalRows failed: {}", ex.getMessage()); } - // 节流 + 异步刷新 totalSize:每 SIZE_REFRESH_EVERY 次 progress 提交一次到独立线程; + // 节流 + 异步刷新 totalSize:每 SIZE_REFRESH_EVERY 次落库提交一次到独立线程; // 上一次还没回来就跳过,绝不阻塞主流(MySQL 真流式 cursor 对客户端读速度敏感) if (progressTick.incrementAndGet() % SIZE_REFRESH_EVERY == 0 && sizeRefreshInFlight.compareAndSet(false, true)) { @@ -1357,16 +1571,36 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut } }); - history.setState(RunState.RUNNING); - historyRepository.save(history); - + request.setCheckpoint(job.getCheckpoint()); + int maxRetries = parseIntOrDefault(executorCfg.get("maxRetries"), 0); + int retryInterval = parseIntOrDefault(executorCfg.get("retryIntervalSeconds"), 30); ExecutorResponse response; try { - response = executorService.start(request); + int retries = 0; + while (true) { + response = executorService.start(request); + final String checkpoint = response.getCheckpoint(); + job.update(history -> history.setCheckpoint(checkpoint)); + if (response.getState() != RunState.FAILURE || !response.getRetryable() || retries >= maxRetries) { + break; + } + retries++; + if (response.getMetrics() != null) { + syncJobManager.onMetrics(job, response.getMetrics(), true); + } + if (!job.transition(RunState.RETRYING, response.getMessage())) { + break; + } + waitForRetry(job, retryInterval); + // 等待期间被取消时转换会被拒绝 / a cancel during the wait makes this move illegal + if (!job.transition(RunState.RUNNING, null)) { + return; + } + job.update(history -> history.setAttempt(history.getAttempt() == null ? 2 : history.getAttempt() + 1)); + request.setCheckpoint(checkpoint); + } } finally { - // 任务一旦真正退出,从 STOPPING 集合移除(不管是正常 / 失败 / 被停止) - stoppingHistoryIds.remove(progressHistory.getId()); // 关闭异步 size 刷新线程,等正在跑的最多 30 秒 sizeRefreshExecutor.shutdown(); try { @@ -1379,43 +1613,48 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut Thread.currentThread().interrupt(); } } - history.setUpdateTime(new Date()); - history.setElapsed((history.getUpdateTime().getTime() - history.getCreateTime().getTime()) / 1000); - history.setMode(QueryMode.SYNC); - history.setCount(response.getCount()); - history.setState(response.getState()); - if (response.getMessage() != null) { - history.setMessage(response.getMessage()); + if (response.getMetrics() != null) { + syncJobManager.onMetrics(job, response.getMetrics(), true); } - if (response.getSuccessful()) { - history.setProcessedCount((long) response.getCount()); - if (history.getTotalCount() == null || history.getTotalCount() < 0L) { - history.setTotalCount((long) response.getCount()); + final ExecutorResponse finished = response; + job.update(history -> { + history.setCount(finished.getCount()); + if (finished.getSuccessful()) { + history.setProcessedCount((long) finished.getCount()); + if (history.getTotalCount() == null || history.getTotalCount() < 0L) { + history.setTotalCount((long) finished.getCount()); + } + history.setProgress(java.math.BigDecimal.valueOf(100.0).setScale(2, java.math.RoundingMode.HALF_UP)); } - history.setProgress(java.math.BigDecimal.valueOf(100.0).setScale(2, java.math.RoundingMode.HALF_UP)); - } - historyRepository.save(history); - // STOPPED(用户主动停止)与 TIMEOUT(超时取消)都是终态:已把最终状态写入 history, - // 无需当作异常抛出 / 也不刷 table metadata - if (response.getState() == RunState.STOPPED || response.getState() == RunState.TIMEOUT) { - return; + }); + RunState finalState = response.getState() != null + ? response.getState() + : (response.getSuccessful() ? RunState.SUCCESS : RunState.FAILURE); + // 只有成功才刷新表元数据,刷新完成后才算成功;STOPPED / TIMEOUT / FAILURE 直接落终态 + if (finalState == RunState.SUCCESS) { + long finalizeStart = System.currentTimeMillis(); + syncJobManager.stage(job, SYNC_STAGE_FINALIZE, "RUNNING", 0); + try { + this.flushTableMetadata( + entity, + outputPlugin, + database, + requireNonNull(output.getOriginConfigure()), + outPlugin + ); + } + catch (RuntimeException ex) { + syncJobManager.stage(job, SYNC_STAGE_FINALIZE, "FAILED", System.currentTimeMillis() - finalizeStart); + throw ex; + } + syncJobManager.stage(job, SYNC_STAGE_FINALIZE, "DONE", System.currentTimeMillis() - finalizeStart); } - Preconditions.checkArgument(response.getSuccessful(), response.getMessage()); - - this.flushTableMetadata( - entity, - outputPlugin, - database, - requireNonNull(output.getOriginConfigure()), - outPlugin - ); + job.transition(finalState, response.getMessage()); }); }, () -> { log.error("Executor [ {} ] not found", entity.getExecutor()); - history.setMessage(String.format("Executor [ %s ] not found", entity.getExecutor())); - history.setState(RunState.FAILURE); - historyRepository.save(history); + job.transition(RunState.FAILURE, String.format("Executor [ %s ] not found", entity.getExecutor())); } ); }, @@ -1425,12 +1664,7 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut } catch (Exception e) { log.warn("Sync data for dataset [ {} ] failed", entity.getName(), e); - history.setState(RunState.FAILURE); - history.setMessage(e.getMessage()); - } - finally { - historyRepository.save(history); - service.shutdownNow(); + job.transition(RunState.FAILURE, e.getMessage()); } return entity; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java new file mode 100644 index 0000000000..34d982f33c --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java @@ -0,0 +1,652 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.convert.ConvertFilter; +import org.devlive.metaforge.convert.ConvertService; +import org.devlive.metaforge.convert.model.ConvertChoice; +import org.devlive.metaforge.convert.model.ConvertOption; +import org.devlive.metaforge.convert.model.ConvertRequest; +import org.devlive.metaforge.convert.model.ConvertResponse; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.service.common.SourceColumns; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.exporter.ExportStatements; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.Response; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Exports tables of one database: chosen fields and rows of each table through an installed + * convert plugin with its declared settings, plus a schema.sql of their structure, packed into + * one ZIP (or left as the single file) by a background job the page follows and downloads. + */ +@Slf4j +@Service +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2", "REC_CATCH_EXCEPTION"}, justification = "Collaborators are Spring-managed singletons; a table's failure of any kind is reported on the table") +public class DatabaseExportServiceImpl + implements DatabaseExportService +{ + private static final String RAW = "NoneConvert"; + private static final String SCHEMA = "schema.sql"; + private static final long KEEP = Duration.ofHours(2).toMillis(); + + private final PluginManager pluginManager; + private final SourceRepository repository; + private final Map jobs = new ConcurrentHashMap<>(); + private final ExecutorService runner = Executors.newCachedThreadPool(task -> { + Thread thread = new Thread(task, "metaforge-export"); + thread.setDaemon(true); + return thread; + }); + + public DatabaseExportServiceImpl(PluginManager pluginManager, SourceRepository repository) + { + this.pluginManager = pluginManager; + this.repository = repository; + } + + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + + /** Jobs belong to the user who started them. */ + private static String currentUser() + { + return String.valueOf(UserDetailsService.getUser().getCode()); + } + + // ---- formats ---- + + @Override + public CommonResponse>> formats() + { + List> formats = new ArrayList<>(); + for (PluginMetadata plugin : pluginManager.getPluginInfos()) { + if (plugin.getType() != PluginType.CONVERT || RAW.equals(plugin.getName())) { + continue; + } + Optional converter = ConvertFilter.filter(pluginManager, plugin.getName()); + if (converter.isEmpty()) { + continue; + } + Map format = new LinkedHashMap<>(); + format.put("name", plugin.getName()); + format.put("version", plugin.getVersion()); + format.put("extension", converter.get().extension()); + format.put("merge", false); + format.put("options", converter.get().options().stream().map(DatabaseExportServiceImpl::option).collect(Collectors.toList())); + formats.add(format); + } + return CommonResponse.success(formats); + } + + private static Map option(ConvertOption option) + { + Map value = new LinkedHashMap<>(); + value.put("name", option.getName()); + value.put("labels", option.getLabels()); + value.put("defaultValue", option.getDefaultValue()); + List> choices = new ArrayList<>(); + for (ConvertChoice choice : option.getChoices()) { + Map item = new LinkedHashMap<>(); + item.put("value", choice.getValue()); + item.put("labels", choice.getLabels()); + choices.add(item); + } + value.put("choices", choices); + return value; + } + + // ---- shared ---- + + /** A source of the export: its plugin service, a raw-row configure and what its database holds. */ + private final class Target + { + private PluginService service; + private SourceEntity entity; + private Plugin plugin; + private ExportStatements statements; + private String database; + /** table -> column -> type, in column order, read through the plugin when a table is first needed. */ + private final Map> columns = new ConcurrentHashMap<>(); + + RuntimeConfigure raw() + { + return entity.toConfigure(RAW, pluginManager, plugin); + } + + /** The table's columns and types; null when it does not exist. */ + Map columns(String table) + { + Map found = columns.computeIfAbsent(table, name -> { + Map types = new LinkedHashMap<>(); + SourceColumns.of(service, raw(), database, name).forEach(column -> types.put(column.getName(), column.getType().toUpperCase(Locale.ROOT))); + return types; + }); + return found.isEmpty() ? null : found; + } + } + + private CommonResponse target(String code, String database, ExportPlan plan) + { + Optional entity = repository.findByCode(code); + if (entity.isEmpty()) { + return CommonResponse.failure(String.format("Resource [ %s ] not found", code)); + } + Optional plugin = pluginManager.getPlugin(entity.get().getType()); + if (plugin.isEmpty()) { + return CommonResponse.failure(String.format("Plugin [ %s ] not found", entity.get().getType())); + } + Target target = new Target(); + target.entity = entity.get(); + target.plugin = plugin.get(); + target.service = plugin.get().getService(PluginService.class); + if (!target.service.managementCapabilities().supports("table:export")) { + return CommonResponse.failure("Source does not support table:export"); + } + if (plan != null && plan.includesStructure() && !target.service.managementCapabilities().supports("table:statement")) { + return CommonResponse.failure("Source does not support table:statement"); + } + target.statements = new ExportStatements(target.service::sqlIdentifier, target.service::sqlLiteral); + target.database = database; + return CommonResponse.success(target); + } + + private static List values(Object row, int count) + { + List values = new ArrayList<>(); + if (row instanceof List) { + ((List) row).forEach(value -> values.add(value == null ? null : String.valueOf(value))); + } + else if (row instanceof JsonNode) { + ((JsonNode) row).elements().forEachRemaining(value -> values.add(value.isNull() ? null : value.asText())); + } + while (values.size() < count) { + values.add(""); + } + return values; + } + + private Optional converter(ExportPlan plan) + { + String format = plan.getFormat() == null ? "" : plan.getFormat(); + return format.isEmpty() || RAW.equals(format) ? Optional.empty() : ConvertFilter.filter(pluginManager, format); + } + + /** Problems of one table in the plan, empty when it can be exported. */ + private static List problems(Target target, ExportPlan.TablePlan table) + { + List problems = new ArrayList<>(); + Map columns = target.columns(table.getName()); + if (columns == null) { + problems.add("Table " + table.getName() + " does not exist"); + return problems; + } + for (String field : table.getFields()) { + if (!columns.containsKey(field)) { + problems.add("Field " + field + " does not exist"); + } + } + for (ExportPlan.Filter filter : table.getFilters()) { + String operator = filter.getOperator() == null ? "" : filter.getOperator().toUpperCase(Locale.ROOT); + if (!columns.containsKey(filter.getColumn())) { + problems.add("Filter field " + filter.getColumn() + " does not exist"); + } + else if (!ExportStatements.OPERATORS.contains(operator)) { + problems.add("Unsupported operator " + filter.getOperator()); + } + else if (!operator.startsWith("IS") && (filter.getValue() == null || filter.getValue().isEmpty())) { + problems.add("Filter on " + filter.getColumn() + " needs a value"); + } + } + return problems; + } + + private static String rule(ExportPlan.TablePlan table) + { + return table.getFilters().isEmpty() ? "all" : "filtered"; + } + + // ---- sample ---- + + @Override + public CommonResponse> sample(String code, String database, ExportPlan plan, String table, int rows) + { + Optional converter = converter(plan); + CommonResponse resolved = target(code, database, null); + if (!Boolean.TRUE.equals(resolved.getStatus())) { + return CommonResponse.failure(String.valueOf(resolved.getMessage())); + } + Target target = resolved.getData(); + ExportPlan.TablePlan chosen = plan.getTables().stream().filter(item -> table.equals(item.getName())).findFirst().orElseGet(() -> { + ExportPlan.TablePlan item = new ExportPlan.TablePlan(); + item.setName(table); + return item; + }); + List problems = problems(target, chosen); + if (!problems.isEmpty()) { + return CommonResponse.failure(problems.get(0)); + } + Response response = target.service.execute(target.raw(), target.statements.select(database, chosen, Math.max(1, Math.min(rows, 20)), 0)); + if (response == null || !Boolean.TRUE.equals(response.getIsSuccessful())) { + return CommonResponse.failure(response == null ? "Reading the rows failed" : response.getMessage()); + } + Map result = new LinkedHashMap<>(); + result.put("headers", response.getHeaders()); + result.put("rows", response.getColumns()); + if (converter.isPresent()) { + File directory = null; + try { + directory = Files.createTempDirectory("metaforge-export-sample").toFile(); + File file = write(converter.get(), plan, target, database, chosen, response, directory); + result.put("file", file.getName()); + result.put("content", FileUtils.readFileToString(file, StandardCharsets.UTF_8)); + } + catch (IOException | RuntimeException error) { + return CommonResponse.failure(error.getMessage() == null ? "The format could not write the sample" : error.getMessage()); + } + finally { + FileUtils.deleteQuietly(directory); + } + } + return CommonResponse.success(result); + } + + /** Writes rows through the plugin into the directory and returns the file it wrote. */ + private static File write(ConvertService converter, ExportPlan plan, Target target, String database, ExportPlan.TablePlan table, Response rows, File directory) + throws IOException + { + Map types = Optional.ofNullable(target.columns(table.getName())).orElse(Map.of()); + ConvertRequest request = new ConvertRequest(); + request.setName(table.getName()); + request.setPath(directory.getAbsolutePath()); + request.setDelimiter("\t"); + request.setHeaders(new ArrayList<>(rows.getHeaders())); + request.setColumns(rows.getColumns()); + request.setTypes(rows.getHeaders().stream().map(header -> types.getOrDefault(header, "")).collect(Collectors.toList())); + request.setOptions(plan.getOptions() == null ? Map.of() : plan.getOptions()); + ConvertResponse written = converter.writer(request); + if (!Boolean.TRUE.equals(written.getSuccessful())) { + throw new IOException(written.getMessage() == null ? "The format could not write " + table.getName() : written.getMessage()); + } + File[] files = directory.listFiles(file -> file.getName().startsWith(table.getName() + ".")); + if (files == null || files.length != 1) { + throw new IOException("The format wrote no file for " + table.getName()); + } + return files[0]; + } + + // ---- check ---- + + @Override + public CommonResponse> check(String code, String database, ExportPlan plan) + { + Optional converter = converter(plan); + if (converter.isEmpty()) { + return CommonResponse.failure("Choose an installed export format"); + } + if (plan.getTables().isEmpty()) { + return CommonResponse.failure("Choose at least one table"); + } + if ("merge".equals(plan.getOrganize())) { + return CommonResponse.failure("The format " + plan.getFormat() + " can not merge tables into one file"); + } + CommonResponse resolved = target(code, database, plan); + if (!Boolean.TRUE.equals(resolved.getStatus())) { + return CommonResponse.failure(String.valueOf(resolved.getMessage())); + } + Target target = resolved.getData(); + String extension = converter.get().extension(); + List> tables = new ArrayList<>(); + int errors = 0; + for (ExportPlan.TablePlan table : plan.getTables()) { + List problems = problems(target, table); + errors += problems.isEmpty() ? 0 : 1; + Map row = new LinkedHashMap<>(); + row.put("name", table.getName()); + row.put("structure", plan.includesStructure()); + row.put("data", true); + row.put("rule", rule(table)); + row.put("file", ExportStatements.fileName(table.getName(), extension)); + row.put("status", problems.isEmpty() ? "pass" : "error"); + row.put("messages", problems); + tables.add(row); + } + List files = tables.stream().map(row -> String.valueOf(row.get("file"))).collect(Collectors.toList()); + if (plan.includesStructure()) { + files.add(SCHEMA); + } + Map result = new LinkedHashMap<>(); + result.put("tables", tables); + result.put("files", files); + result.put("output", output(plan, files.size(), extension, plan.getTables().get(0).getName())); + result.put("errors", errors); + return CommonResponse.success(result); + } + + /** One file without compression is downloaded as it is; otherwise everything goes into a ZIP. */ + private static String output(ExportPlan plan, int files, String extension, String table) + { + String base = ExportStatements.safeName(plan.getFileName(), "export"); + return files == 1 && !plan.isCompress() ? ExportStatements.fileName(ExportStatements.safeName(table, base), extension) : base + ".zip"; + } + + // ---- execute ---- + + /** A running or finished export; tables keep their progress, the file is written when all are done. */ + private static final class Job + { + private final String id = UUID.randomUUID().toString(); + private final String user; + private final long created = System.currentTimeMillis(); + private final boolean keep; + private final Map> tables = new LinkedHashMap<>(); + private final AtomicBoolean stopped = new AtomicBoolean(); + private volatile String status = "running"; + private volatile String message; + private volatile File file; + private volatile String name; + private volatile File directory; + + private Job(String user, boolean keep) + { + this.user = user; + this.keep = keep; + } + } + + @Override + public CommonResponse> execute(String code, String database, ExportPlan plan) + { + CommonResponse> checked = check(code, database, plan); + if (!Boolean.TRUE.equals(checked.getStatus())) { + return checked; + } + if (((Number) checked.getData().get("errors")).intValue() > 0) { + return CommonResponse.failure("Fix the tables that do not pass the check first"); + } + ConvertService converter = converter(plan).orElseThrow(); + Target target = target(code, database, plan).getData(); + expire(); + Job job = new Job(currentUser(), plan.isKeepRecord()); + for (ExportPlan.TablePlan table : plan.getTables()) { + Map item = new ConcurrentHashMap<>(); + item.put("name", table.getName()); + item.put("status", "waiting"); + item.put("written", 0L); + item.put("total", -1L); + job.tables.put(table.getName(), item); + } + jobs.put(job.id, job); + String output = String.valueOf(checked.getData().get("output")); + runner.submit(() -> { + run(job, target, converter, database, plan, output); + notifyFinished(job, code, database); + }); + Map result = new LinkedHashMap<>(); + result.put("jobId", job.id); + return CommonResponse.success(result); + } + + private void run(Job job, Target target, ConvertService converter, String database, ExportPlan plan, String output) + { + Map schemas = new ConcurrentHashMap<>(); + ExecutorService pool = Executors.newFixedThreadPool(Math.max(1, Math.min(4, plan.getConcurrency()))); + try { + job.directory = Files.createTempDirectory("metaforge-export").toFile(); + for (ExportPlan.TablePlan table : plan.getTables()) { + pool.submit(() -> exportTable(job, target, converter, database, plan, table, schemas)); + } + pool.shutdown(); + if (!pool.awaitTermination(6, TimeUnit.HOURS)) { + pool.shutdownNow(); + job.stopped.set(true); + job.message = "The export took longer than six hours"; + } + if (job.stopped.get()) { + job.status = "failed"; + job.message = job.message == null ? "The export stopped at a failing table" : job.message; + return; + } + if (plan.includesStructure()) { + String schema = plan.getTables().stream().map(table -> schemas.getOrDefault(table.getName(), "")).collect(Collectors.joining("\n")); + FileUtils.writeStringToFile(new File(job.directory, SCHEMA), schema, StandardCharsets.UTF_8); + } + job.file = pack(job.directory, plan, output); + job.name = job.file.getParentFile().equals(job.directory) ? job.file.getName() : output; + boolean failed = job.tables.values().stream().anyMatch(item -> "failed".equals(item.get("status"))); + job.status = failed ? "partial" : "done"; + } + catch (IOException | RuntimeException error) { + log.warn("Export {} failed", job.id, error); + job.status = "failed"; + job.message = error.getMessage(); + } + catch (InterruptedException error) { + Thread.currentThread().interrupt(); + job.status = "failed"; + job.message = "The export was interrupted"; + } + finally { + pool.shutdownNow(); + } + } + + private void notifyFinished(Job job, String code, String database) + { + boolean succeeded = "done".equals(job.status); + long failed = job.tables.values().stream().filter(item -> "failed".equals(item.get("status"))).count(); + String content = succeeded + ? String.format("%s 的 %d 张表已导出为 %s,可在导出页面下载。", database, job.tables.size(), job.name) + : String.format("%s 导出未完成:%s", database, job.message != null ? job.message : failed + " 张表导出失败"); + notifications.publishTo(job.user, succeeded ? NotificationScene.EXPORT_SUCCEEDED : NotificationScene.EXPORT_FAILED, + String.format(succeeded ? "%s 导出完成" : "%s 导出失败", database), content, EntityType.SOURCE, code, database); + } + + private void exportTable(Job job, Target target, ConvertService converter, String database, ExportPlan plan, ExportPlan.TablePlan table, Map schemas) + { + Map item = job.tables.get(table.getName()); + if (job.stopped.get()) { + item.put("status", "cancelled"); + return; + } + item.put("status", "running"); + RuntimeConfigure configure = target.raw(); + try { + if (plan.includesStructure()) { + // The plugin builds the CREATE statement, as the DDL tab shows it. + Response create = target.service.getTableDdl(configure, database, table.getName(), "create"); + if (create == null || !Boolean.TRUE.equals(create.getIsSuccessful()) || create.getContent() == null) { + throw new IOException(create == null ? "Reading the structure failed" : create.getMessage()); + } + String statement = create.getContent(); + schemas.put(table.getName(), target.statements.schema(database, table.getName(), statement, plan.getStructure())); + } + Response total = target.service.execute(configure, target.statements.count(database, table)); + if (total != null && Boolean.TRUE.equals(total.getIsSuccessful()) && !total.getColumns().isEmpty()) { + item.put("total", parseLong(values(total.getColumns().get(0), 1).get(0))); + } + Response rows = read(job, target, configure, database, plan, table, item); + write(converter, plan, target, database, table, rows, job.directory); + item.put("status", "done"); + } + catch (IOException | RuntimeException error) { + log.warn("Export of {}.{} failed", database, table.getName(), error); + item.put("status", "failed"); + item.put("message", error.getMessage() == null ? "Export failed" : error.getMessage()); + if ("stop".equals(plan.getFailure())) { + job.stopped.set(true); + job.message = table.getName() + ": " + item.get("message"); + } + } + } + + /** Every row of a table, read in pages of batchSize (or in one query), with the written count kept current. */ + private static Response read(Job job, Target target, RuntimeConfigure configure, String database, ExportPlan plan, ExportPlan.TablePlan table, Map item) + throws IOException + { + long batch = "single".equals(plan.getReading()) ? 0 : Math.max(1, Math.min(100_000, plan.getBatchSize())); + Response all = null; + List rows = new ArrayList<>(); + for (long offset = 0; ; offset += batch) { + if (job.stopped.get()) { + throw new IOException("Stopped"); + } + Response page = target.service.execute(configure, target.statements.select(database, table, batch, offset)); + if (page == null || !Boolean.TRUE.equals(page.getIsSuccessful())) { + throw new IOException(page == null ? "Reading the rows failed" : page.getMessage()); + } + all = all == null ? page : all; + rows.addAll(page.getColumns()); + item.put("written", (long) rows.size()); + if (batch == 0 || page.getColumns().size() < batch) { + break; + } + } + all.setColumns(rows); + return all; + } + + private static File pack(File directory, ExportPlan plan, String output) + throws IOException + { + File[] files = directory.listFiles(); + if (files == null || files.length == 0) { + throw new IOException("The export wrote no file"); + } + Arrays.sort(files); + if (!output.endsWith(".zip") && files.length == 1) { + return files[0]; + } + File zip = new File(directory.getParentFile(), directory.getName() + ".zip"); + try (OutputStream out = Files.newOutputStream(zip.toPath()); ZipOutputStream stream = new ZipOutputStream(out)) { + stream.setLevel(plan.isCompress() ? Deflater.DEFAULT_COMPRESSION : Deflater.NO_COMPRESSION); + for (File file : files) { + stream.putNextEntry(new ZipEntry(file.getName())); + Files.copy(file.toPath(), stream); + stream.closeEntry(); + } + } + return zip; + } + + private static long parseLong(String value) + { + if (value == null) { + return -1; + } + try { + return Long.parseLong(value.trim()); + } + catch (NumberFormatException error) { + return -1; + } + } + + // ---- job ---- + + private Job owned(String id) + { + Job job = jobs.get(id); + return job == null || !job.user.equals(currentUser()) ? null : job; + } + + @Override + public CommonResponse> job(String id) + { + Job job = owned(id); + if (job == null) { + return CommonResponse.failure("Export job not found"); + } + Map result = new LinkedHashMap<>(); + result.put("status", job.status); + result.put("message", job.message); + result.put("tables", new ArrayList<>(job.tables.values())); + File file = job.file; + if (file != null) { + result.put("file", job.name); + result.put("size", file.length()); + } + return CommonResponse.success(result); + } + + @Override + public CommonResponse file(String id) + { + Job job = owned(id); + if (job == null || job.file == null) { + return CommonResponse.failure("The export file is not ready"); + } + try { + TableExportService.ExportFile file = new TableExportService.ExportFile(job.name, FileUtils.readFileToByteArray(job.file)); + if (!job.keep) { + remove(job); + } + return CommonResponse.success(file); + } + catch (IOException error) { + return CommonResponse.failure(error.getMessage()); + } + } + + /** Drops finished jobs older than two hours with their files. */ + private void expire() + { + long now = System.currentTimeMillis(); + jobs.values().stream().filter(job -> !"running".equals(job.status) && now - job.created > KEEP).collect(Collectors.toList()).forEach(this::remove); + } + + private void remove(Job job) + { + jobs.remove(job.id); + if (job.file != null && job.directory != null && !job.file.getParentFile().equals(job.directory)) { + FileUtils.deleteQuietly(job.file); + } + FileUtils.deleteQuietly(job.directory); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/NotificationServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/NotificationServiceImpl.java deleted file mode 100644 index d6f4e85b08..0000000000 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/NotificationServiceImpl.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.devlive.metaforge.service.service.impl; - -import org.devlive.metaforge.service.service.NotificationService; -import org.springframework.stereotype.Service; - -@Service -public class NotificationServiceImpl - implements NotificationService -{ -} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/OAuthServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/OAuthServiceImpl.java new file mode 100644 index 0000000000..c073b5ee3b --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/OAuthServiceImpl.java @@ -0,0 +1,279 @@ +package org.devlive.metaforge.service.service.impl; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.entity.UserOAuthEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; +import org.devlive.metaforge.service.repository.UserOAuthRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.OAuthService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.settings.OAuthClient; +import org.devlive.metaforge.service.settings.OAuthProvider; +import org.devlive.metaforge.service.settings.OAuthSettings; +import org.devlive.metaforge.service.settings.OAuthStates; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClientException; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@Slf4j +@Service +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class OAuthServiceImpl + implements OAuthService +{ + /** The web page that finishes the sign-in; the console routes by the hash. */ + static final String CALLBACK_PAGE = "/auth/oauth"; + private static final Pattern NOT_NAME = Pattern.compile("[^A-Za-z0-9_]+"); + private static final int USERNAME_BASE = 15; + + private final SystemSettingService settings; + private final OAuthStates states; + private final OAuthClient client; + private final UserService userService; + private final UserRepository userRepository; + private final UserOAuthRepository links; + private final SecureRandom random = new SecureRandom(); + + public OAuthServiceImpl(SystemSettingService settings, OAuthStates states, OAuthClient client, UserService userService, UserRepository userRepository, + UserOAuthRepository links) + { + this.settings = settings; + this.states = states; + this.client = client; + this.userService = userService; + this.userRepository = userRepository; + this.links = links; + } + + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + + /** A sign-in that stops with an error code the callback page explains. */ + private static final class Failure + extends RuntimeException + { + private static final long serialVersionUID = 1L; + + private Failure(String code) + { + super(code); + } + } + + /** The address to register at the provider. */ + public static String callbackUrl(String base, OAuthProvider provider) + { + return base + "/api/auth/oauth/" + provider.getId() + "/callback"; + } + + private String base(String request) + { + String site = settings.auth().getSiteUrl(); + return StringUtils.stripEnd(StringUtils.isNotBlank(site) ? site : StringUtils.defaultString(request), "/"); + } + + /** Only a path on this site, so the sign-in can not be used to send someone elsewhere. */ + static String safeRedirect(String value) + { + if (StringUtils.isBlank(value) || value.length() > 2000 || !value.startsWith("/") || value.startsWith("//") || value.contains("\\") + || value.chars().anyMatch(c -> c < 0x20 || c == 0x7f)) { + return ""; + } + return value; + } + + private static String page(String base, String key, String value, String redirect) + { + Map query = new LinkedHashMap<>(); + query.put(key, value); + if (StringUtils.isNotEmpty(redirect)) { + query.put("redirect", redirect); + } + return base + CALLBACK_PAGE + "?" + query(query); + } + + private static String query(Map values) + { + return values.entrySet().stream() + .map(entry -> URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) + "=" + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)) + .collect(Collectors.joining("&")); + } + + @Override + public Redirect authorize(String provider, String redirect, String request) + { + String base = base(request); + String target = safeRedirect(redirect); + Optional found = OAuthProvider.of(provider); + OAuthSettings configured = found.map(settings::oauth).orElse(null); + if (configured == null || !configured.ready()) { + return new Redirect(page(base, "error", "disabled", target), null); + } + OAuthStates.Pending pending = states.begin(found.get(), target); + Map query = new LinkedHashMap<>(); + query.put("response_type", "code"); + query.put("client_id", configured.getClientId()); + query.put("redirect_uri", callbackUrl(base, found.get())); + query.put("scope", configured.getScopes()); + query.put("state", pending.getState()); + query.put("code_challenge", OAuthStates.challenge(pending.getVerifier())); + query.put("code_challenge_method", "S256"); + String endpoint = configured.getAuthorizationUrl(); + return new Redirect(endpoint + (endpoint.contains("?") ? "&" : "?") + query(query), pending.getState()); + } + + @Override + public String callback(String provider, String code, String state, String browserState, String error, String request) + { + String base = base(request); + OAuthStates.Pending pending = states.take(state).orElse(null); + OAuthProvider found = OAuthProvider.of(provider).orElse(null); + // The state must be one this site issued, for this provider, to this browser. + if (pending == null || found == null || pending.getProvider() != found || browserState == null + || !MessageDigest.isEqual(browserState.getBytes(StandardCharsets.UTF_8), state.getBytes(StandardCharsets.UTF_8))) { + return page(base, "error", "state", ""); + } + String redirect = pending.getRedirect(); + if (StringUtils.isNotBlank(error)) { + return page(base, "error", "denied", redirect); + } + OAuthSettings configured = settings.oauth(found); + if (!configured.ready()) { + return page(base, "error", "disabled", redirect); + } + if (StringUtils.isBlank(code)) { + return page(base, "error", "failed", redirect); + } + try { + OAuthClient.Profile profile = client.profile(found, configured, code, callbackUrl(base, found), pending.getVerifier()); + UserEntity user = user(found, profile); + CommonResponse issued = userService.issue(user.getUsername(), 0); + if (!Boolean.TRUE.equals(issued.getStatus()) || issued.getData() == null) { + return page(base, "error", "failed", redirect); + } + return page(base, "ticket", states.ticket(issued.getData()), redirect); + } + catch (Failure e) { + return page(base, "error", e.getMessage(), redirect); + } + catch (RestClientException e) { + log.warn("Signing in with {} failed: {}", found.getLabel(), e.getMessage()); + return page(base, "error", "failed", redirect); + } + catch (RuntimeException e) { + log.warn("Signing in with {} failed", found.getLabel(), e); + return page(base, "error", "failed", redirect); + } + } + + /** The linked user, else the user with the verified email, else a new user while registration is open. */ + private UserEntity user(OAuthProvider provider, OAuthClient.Profile profile) + { + Optional link = links.findByProviderAndSubject(provider.getId(), profile.getSubject()); + Optional linked = link.flatMap(value -> userRepository.findById(value.getUserId())); + if (linked.isPresent()) { + return active(linked.get()); + } + String email = AuthServiceImpl.isEmail(profile.getEmail()) ? profile.getEmail().trim().toLowerCase(Locale.ROOT) : ""; + UserEntity user = email.isEmpty() ? null : userRepository.findByEmail(email).map(this::active).orElse(null); + boolean existing = user != null; + if (user == null) { + if (!settings.auth().isRegistration()) { + throw new Failure("registration"); + } + user = create(provider, profile, email); + } + // A link left by a removed user is taken over. + links.save(UserOAuthEntity.builder() + .id(link.map(UserOAuthEntity::getId).orElse(null)) + .userId(user.getId()) + .provider(provider.getId()) + .subject(profile.getSubject()) + .createTime(new Date()) + .build()); + if (existing) { + // Off the request's security context: the user is only now signing in. + notifications.publishTo(user.getCode(), NotificationScene.OAUTH_BOUND, String.format("已绑定 %s 账号", provider.getLabel()), + String.format("你的账号已通过邮箱 %s 与 %s 账号关联,之后可以直接用它登录;如非本人操作,请联系管理员。", email, provider.getLabel()), + EntityType.USER, user.getCode(), user.getUsername()); + } + return user; + } + + private UserEntity active(UserEntity user) + { + if (!user.isActive()) { + throw new Failure("inactive"); + } + return user; + } + + private UserEntity create(OAuthProvider provider, OAuthClient.Profile profile, String email) + { + UserEntity user = new UserEntity(); + user.setUsername(username(provider, profile, email)); + // Signing in with a password needs a reset first; the provider is how this user signs in. + user.setPassword(UUID.randomUUID().toString()); + if (!email.isEmpty()) { + user.setEmail(email); + } + CommonResponse saved = userService.saveOrUpdate(user); + if (!Boolean.TRUE.equals(saved.getStatus()) || saved.getData() == null) { + throw new Failure("failed"); + } + return saved.getData(); + } + + /** The provider login made fit for a username, with a number added while it is taken. */ + private String username(OAuthProvider provider, OAuthClient.Profile profile, String email) + { + String source = StringUtils.defaultString(StringUtils.firstNonBlank(profile.getLogin(), StringUtils.substringBefore(email, "@"))); + String name = StringUtils.left(StringUtils.strip(NOT_NAME.matcher(source).replaceAll("_"), "_"), USERNAME_BASE); + if (name.length() < 3) { + name = provider.getId() + "_user"; + } + if (userRepository.findByUsername(name).isEmpty()) { + return name; + } + for (int i = 0; i < 20; i++) { + String candidate = name + "_" + (1000 + random.nextInt(9000)); + if (userRepository.findByUsername(candidate).isEmpty()) { + return candidate; + } + } + throw new Failure("failed"); + } + + @Override + public CommonResponse redeem(String ticket) + { + return states.redeem(ticket) + .map(CommonResponse::success) + .orElseGet(() -> CommonResponse.failure("The sign-in has expired; sign in again")); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/PluginAuditServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/PluginAuditServiceImpl.java index c67fba1d53..ab4ad9ccc6 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/PluginAuditServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/PluginAuditServiceImpl.java @@ -12,6 +12,7 @@ import org.devlive.metaforge.fs.FsService; import org.devlive.metaforge.plugin.PluginManager; import org.devlive.metaforge.service.activity.HeatmapActivity; +import org.devlive.metaforge.service.activity.QueryActivity; import org.devlive.metaforge.service.entity.PluginAuditEntity; import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.initializer.InitializerConfigure; @@ -20,6 +21,7 @@ import org.devlive.metaforge.service.repository.PluginAuditRepository; import org.devlive.metaforge.service.security.UserDetailsService; import org.devlive.metaforge.service.service.PluginAuditService; +import org.devlive.metaforge.service.service.SystemSettingService; import org.devlive.metaforge.spi.model.Response; import org.springframework.jdbc.core.RowCallbackHandler; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; @@ -32,7 +34,9 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -44,14 +48,16 @@ public class PluginAuditServiceImpl implements PluginAuditService { + private final SystemSettingService systemSettings; private final PluginAuditRepository pluginAuditRepository; private final InitializerConfigure initializer; private final PluginManager pluginManager; private final NamedParameterJdbcOperations jdbc; public PluginAuditServiceImpl(PluginAuditRepository pluginAuditRepository, InitializerConfigure initializer, PluginManager pluginManager, - NamedParameterJdbcOperations jdbc) + NamedParameterJdbcOperations jdbc, SystemSettingService systemSettings) { + this.systemSettings = systemSettings; this.jdbc = jdbc; this.pluginAuditRepository = pluginAuditRepository; this.initializer = initializer; @@ -64,6 +70,45 @@ public CommonResponse count() return CommonResponse.success(this.pluginAuditRepository.countByUserId(UserDetailsService.getUser().getId())); } + @Override + public CommonResponse getHomeActivity(int days) + { + if (days != 7 && days != 30) { + return CommonResponse.failure("Unsupported activity period"); + } + Long userId = UserDetailsService.getUser().getId(); + if (userId == null) { + return CommonResponse.failure("Query activity not found"); + } + LocalDate today = LocalDate.now(); + LocalDate start = today.minusDays(days - 1L); + Map trend = new LinkedHashMap<>(); + for (int day = 0; day < days; day++) { + trend.put(start.plusDays(day).toString(), 0L); + } + Map sources = new HashMap<>(); + Map parameters = new HashMap<>(); + parameters.put("userId", userId); + parameters.put("from", Timestamp.valueOf(start.atStartOfDay())); + parameters.put("until", Timestamp.valueOf(today.plusDays(1).atStartOfDay())); + // Date grouping in Java keeps the same calendar boundaries on every metadata database. + jdbc.query("SELECT q.create_time, s.name FROM metaforge_source_query q " + + "LEFT JOIN metaforge_source s ON q.source_id = s.id " + + "WHERE q.user_id = :userId AND q.create_time >= :from AND q.create_time < :until", + parameters, (RowCallbackHandler) row -> { + String date = row.getTimestamp(1).toLocalDateTime().toLocalDate().toString(); + trend.computeIfPresent(date, (key, count) -> count + 1); + String name = row.getString(2); + sources.merge(name == null ? "" : name, 1L, Long::sum); + }); + List points = trend.entrySet().stream() + .map(entry -> new QueryActivity.Day(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + List distribution = sources.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()).thenComparing(Map.Entry.comparingByKey())) + .map(entry -> new QueryActivity.Source(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + return CommonResponse.success(new QueryActivity(days, sources.values().stream().mapToLong(Long::longValue).sum(), points, distribution)); + } + @Override public CommonResponse> getAllContribution() { @@ -125,18 +170,18 @@ public CommonResponse getData(String code) .map(value -> { Response response = new Response(); FsRequest fsRequest = FsRequest.builder() - .access(initializer.getFsConfigure().getAccess()) - .secret(initializer.getFsConfigure().getSecret()) + .access(systemSettings.fs().getAccess()) + .secret(systemSettings.fs().getSecret()) .endpoint(value.getHome()) - .bucket(initializer.getFsConfigure().getBucket()) + .bucket(systemSettings.fs().getBucket()) .fileName("result") .build(); // If it is OSS third-party storage, rebuild the default directory - if (!initializer.getFsConfigure().getType().equals("LocalFs")) { - fsRequest.setEndpoint(initializer.getFsConfigure().getEndpoint()); + if (!systemSettings.fs().getType().equals("LocalFs")) { + fsRequest.setEndpoint(systemSettings.fs().getEndpoint()); fsRequest.setFileName(String.join(File.separator, value.getUser().getCode(), DateUtils.formatYMD(), String.join(File.separator, "adhoc", code), "result")); } - pluginManager.getPlugin(initializer.getFsConfigure().getType()) + pluginManager.getPlugin(systemSettings.fs().getType()) .ifPresent(plugin -> { FsService fsService = plugin.getService(FsService.class); FsResponse fsResponse = fsService.reader(fsRequest); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/QueryChartRefresh.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/QueryChartRefresh.java new file mode 100644 index 0000000000..f423abb336 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/QueryChartRefresh.java @@ -0,0 +1,159 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.enums.ReportType; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.column.JdbcColumn; +import org.springframework.stereotype.Component; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +@Component +final class QueryChartRefresh +{ + private static final int MAX_ROWS = 10000; + private static final int TIMEOUT_SECONDS = 15; + + private final ObjectMapper mapper; + private final PluginManager plugins; + + QueryChartRefresh(ObjectMapper mapper, PluginManager plugins) + { + this.mapper = mapper; + this.plugins = plugins; + } + + JsonNode chart(ReportEntity report) + { + if (report.getType() != ReportType.QUERY || !report.isRealtime() || report.getConfigure() == null) { + return null; + } + try { + JsonNode configuration = mapper.readTree(report.getConfigure()); + if (configuration == null) { + return null; + } + JsonNode chart = configuration.path("queryChart"); + return chart.isMissingNode() ? null : chart; + } + catch (JsonProcessingException e) { + return null; + } + } + + String validationError(ReportEntity report, SourceEntity source, Long userId, JsonNode chart) + { + if (!source.isActive() || !Objects.equals(source.getUserId(), userId)) { + return "Dynamic refresh requires an active data source that you own."; + } + if (source.isUsedConfig() || !Set.of("mysql", "postgresql").contains(engine(source))) { + return "Dynamic refresh requires a standard MySQL or PostgreSQL connection."; + } + if (chart.path("version").asInt() != 1 || fields(chart).isEmpty()) { + return "Invalid dynamic chart fields."; + } + if (!ReadOnlyChartSql.accepts(report.getQuery())) { + return "Dynamic refresh requires one read-only SELECT with supported SQL functions."; + } + return null; + } + + List> execute(ReportEntity report, SourceEntity source, JsonNode chart) + throws SQLException + { + var plugin = plugins.getPlugin(source.getType()).orElseThrow(() -> new SQLException("Source plugin unavailable")); + var service = plugin.getService(PluginService.class); + try { + service.connect(source.toConfigure(plugins, plugin)); + var connected = PluginService.local.get(); + if (connected == null || !(connected.getConnection() instanceof Connection)) { + throw new SQLException("Source connection unavailable"); + } + Connection connection = (Connection) connected.getConnection(); + boolean mysql = "mysql".equals(engine(source)); + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.setQueryTimeout(TIMEOUT_SECONDS); + statement.execute(mysql ? "START TRANSACTION READ ONLY" : "SET TRANSACTION READ ONLY"); + statement.setMaxRows(Math.max(1, Math.min(MAX_ROWS, chart.path("options").path("maxRows").asInt(MAX_ROWS)))); + try (ResultSet result = statement.executeQuery(report.getQuery())) { + return readRows(result, fields(chart)); + } + } + finally { + connection.rollback(); + } + } + finally { + service.destroy(); + } + } + + private static Set fields(JsonNode chart) + { + JsonNode nodes = chart.path("options").path("fields"); + Set fields = new LinkedHashSet<>(); + if (nodes.isArray() && nodes.size() <= 1024) { + for (JsonNode field : nodes) { + if (!field.isTextual() || field.textValue().isBlank() || !fields.add(field.textValue())) { + return Set.of(); + } + } + } + return fields; + } + + private static String engine(SourceEntity source) + { + return Objects.toString(source.getType(), "").toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", ""); + } + + private static List> readRows(ResultSet result, Set fields) + throws SQLException + { + var metadata = result.getMetaData(); + Map columns = new LinkedHashMap<>(); + for (int column = 1; column <= metadata.getColumnCount(); column++) { + String name = metadata.getColumnLabel(column); + if (fields.contains(name) && columns.putIfAbsent(name, column) != null) { + throw new SQLException("Duplicate chart field"); + } + } + if (!columns.keySet().containsAll(fields)) { + throw new SQLException("Missing chart field"); + } + + JdbcColumn reader = new JdbcColumn(result); + List> rows = new ArrayList<>(); + while (result.next()) { + Map row = new LinkedHashMap<>(); + for (String field : fields) { + int column = columns.get(field); + Object value = reader.mappingColumnData(metadata.getColumnTypeName(column), column); + if (!(value == null || value instanceof CharSequence || value instanceof Number + || value instanceof Boolean || value instanceof java.util.Date || value instanceof byte[])) { + value = result.getString(column); + } + row.put(field, result.wasNull() ? null : value); + } + rows.add(row); + } + return rows; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSql.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSql.java new file mode 100644 index 0000000000..3bab4537c1 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSql.java @@ -0,0 +1,94 @@ +package org.devlive.metaforge.service.service.impl; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** Conservative SQL subset for unattended chart refresh. The database also enforces a read-only transaction. */ +final class ReadOnlyChartSql +{ + private static final Set BLOCKED = Set.of("INTO", "FOR", "LOCK", "OUTFILE", "DUMPFILE", "PROCEDURE", "OPERATOR"); + private static final Set CALLS = Set.of( + "SELECT", "FROM", "AS", "WHERE", "HAVING", "ON", "AND", "OR", "NOT", "IN", "EXISTS", "BY", + "WHEN", "THEN", "ELSE", "OVER", "FILTER", "UNION", "INTERSECT", "EXCEPT", "JOIN", + "COUNT", "SUM", "MIN", "MAX", "AVG", "COALESCE", "NULLIF", "ABS", "ROUND", "CEIL", "FLOOR", + "LOWER", "UPPER", "LENGTH", "CHAR_LENGTH", "CONCAT", "TRIM", "SUBSTRING", "REPLACE", + "DATE", "DATE_TRUNC", "DATE_FORMAT", "EXTRACT", "CAST", "DECIMAL", "NUMERIC", "CHAR", "VARCHAR", + "ROW_NUMBER", "RANK", "DENSE_RANK", "LAG", "LEAD"); + + private ReadOnlyChartSql() {} + + static boolean accepts(String sql) + { + if (sql == null || sql.length() > 32768 || sql.indexOf('\\') >= 0 || sql.indexOf(0) >= 0) { + return false; + } + + List tokens = new ArrayList<>(); + for (int i = 0; i < sql.length(); ) { + char ch = sql.charAt(i); + char next = i + 1 < sql.length() ? sql.charAt(i + 1) : 0; + if (Character.isWhitespace(ch)) { + i++; + } + else if (ch == ';') { + if (!sql.substring(i + 1).isBlank()) { + return false; + } + break; + } + else if (ch == '\'' || ch == '"' || ch == '`') { + boolean closed = false; + for (i++; i < sql.length(); i++) { + if (sql.charAt(i) == ch) { + if (i + 1 < sql.length() && sql.charAt(i + 1) == ch) { + i++; + continue; + } + i++; + closed = true; + break; + } + } + if (!closed) { + return false; + } + tokens.add(ch == '\'' ? "literal" : "identifier"); + } + else if (ch == '#' || ch == '$' || ch == '@' || (ch == '-' && next == '-') || (ch == '/' && next == '*')) { + return false; + } + else if (Character.isLetter(ch) || ch == '_') { + int start = i++; + while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_')) { + i++; + } + String word = sql.substring(start, i).toUpperCase(Locale.ROOT); + if (BLOCKED.contains(word)) { + return false; + } + tokens.add(word); + } + else { + tokens.add(String.valueOf(ch)); + i++; + } + } + + if (tokens.size() < 2 || !"SELECT".equals(tokens.get(0))) { + return false; + } + for (int i = 1; i + 1 < tokens.size(); i++) { + String word = tokens.get(i); + if ("(".equals(tokens.get(i + 1)) && Character.isLetter(word.charAt(0)) + && (!CALLS.contains(word) || ".".equals(tokens.get(i - 1)))) { + return false; + } + if ("(".equals(tokens.get(i + 1)) && word.charAt(0) == '_') { + return false; + } + } + return true; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java index a6bc5d3866..2c85953e81 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java @@ -1,10 +1,15 @@ package org.devlive.metaforge.service.service.impl; +import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.body.FilterBody; import org.devlive.metaforge.service.entity.PageEntity; import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; import org.devlive.metaforge.service.persistence.Persistence; import org.devlive.metaforge.service.repository.BaseRepository; import org.devlive.metaforge.service.repository.DataSetRepository; @@ -13,27 +18,49 @@ import org.devlive.metaforge.service.repository.jdbc.LinkRepository; import org.devlive.metaforge.service.security.UserDetailsService; import org.devlive.metaforge.service.service.ReportService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + @Service @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") public class ReportServiceImpl implements ReportService { + private static final Logger LOG = LoggerFactory.getLogger(ReportServiceImpl.class); + + private final QueryChartRefresh chartRefresh; private final ReportRepository repository; private final SourceRepository sourceRepository; private final DataSetRepository dataSetRepository; private final LinkRepository linkRepository; - public ReportServiceImpl(ReportRepository repository, SourceRepository sourceRepository, DataSetRepository dataSetRepository, LinkRepository linkRepository) + public ReportServiceImpl(ReportRepository repository, SourceRepository sourceRepository, DataSetRepository dataSetRepository, + LinkRepository linkRepository, QueryChartRefresh chartRefresh) { + this.chartRefresh = chartRefresh; this.linkRepository = linkRepository; this.repository = repository; this.sourceRepository = sourceRepository; this.dataSetRepository = dataSetRepository; } + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + @Override public CommonResponse> getAll(FilterBody filter) { @@ -43,31 +70,106 @@ public CommonResponse> getAll(FilterBody filter) @Override public CommonResponse saveOrUpdate(BaseRepository repository, ReportEntity configure) { + boolean created = configure.getId() == null; + CommonResponse response = save(repository, configure); + if (created && Boolean.TRUE.equals(response.getStatus()) && response.getData() != null) { + ReportEntity report = response.getData(); + notifications.publish(NotificationScene.REPORT_CREATED, String.format("报表「%s」已创建", report.getName()), + String.format("报表「%s」已保存。", report.getName()), EntityType.REPORT, report.getCode(), report.getName()); + } + return response; + } + + private CommonResponse save(BaseRepository repository, ReportEntity configure) + { + var user = UserDetailsService.getUser(); + Optional existing = configure.getId() == null ? Optional.empty() : this.repository.findById(configure.getId()) + .filter(value -> Objects.equals(value.getUserId(), user.getId())); + if (user.getId() == null || (configure.getId() != null && existing.isEmpty())) { + return CommonResponse.failure("Report not found"); + } + // Sharing changes only through the share endpoint: a save keeps the stored values, and a new report starts private. + configure.setShareToken(existing.map(ReportEntity::getShareToken).orElse(null)); + configure.setShareEnabled(existing.map(ReportEntity::isShareEnabled).orElse(false)); + configure.setShareEmbed(existing.map(ReportEntity::isShareEmbed).orElse(false)); + configure.setShareOrigins(existing.map(ReportEntity::getShareOrigins).orElse(null)); + configure.setShareExpiresAt(existing.map(ReportEntity::getShareExpiresAt).orElse(null)); + configure.setShareShowTitle(existing.map(ReportEntity::isShareShowTitle).orElse(true)); + configure.setShareShowDescription(existing.map(ReportEntity::isShareShowDescription).orElse(true)); + configure.setShareAllowData(existing.map(ReportEntity::isShareAllowData).orElse(false)); + configure.setShareAllowDownload(existing.map(ReportEntity::isShareAllowDownload).orElse(false)); + JsonNode chart = chartRefresh.chart(configure); + if (chart != null && configure.getSource() == null) { + return CommonResponse.failure("Dynamic chart data source not found"); + } if (configure.getSource() != null) { return sourceRepository.findByCode(configure.getSource().getCode()) .map(value -> { - configure.setUser(UserDetailsService.getUser()); + if (chart != null) { + String error = chartRefresh.validationError(configure, value, user.getId(), chart); + if (error != null) { + return CommonResponse.failure(error); + } + } + configure.setUser(user); configure.setSource(value); return CommonResponse.success(repository.save(configure)); }) .orElseGet(() -> CommonResponse.failure(String.format("Source [ %s ] not found", configure.getSource().getCode()))); } - else { + else if (configure.getDataset() != null) { return dataSetRepository.findByCode(configure.getDataset().getCode()) .map(value -> { - configure.setUser(UserDetailsService.getUser()); + configure.setUser(user); configure.setDataset(value); return CommonResponse.success(repository.save(configure)); }) .orElseGet(() -> CommonResponse.failure(String.format("Source [ %s ] not found", configure.getSource().getCode()))); } + return CommonResponse.failure("Report data source not found"); + } + + @Override + public CommonResponse>> refreshChart(String code) + { + Long userId = UserDetailsService.getUser().getId(); + if (userId == null) { + return CommonResponse.failure("Dynamic chart not found"); + } + ReportEntity report = repository.findByCode(code) + .filter(value -> value.isActive() && Objects.equals(value.getUserId(), userId)) + .orElse(null); + if (report == null || report.getSourceId() == null) { + return CommonResponse.failure("Dynamic chart not found"); + } + JsonNode chart = chartRefresh.chart(report); + SourceEntity source = sourceRepository.findById(report.getSourceId()).orElse(null); + if (chart == null || source == null) { + return CommonResponse.failure("Dynamic chart not found"); + } + String error = chartRefresh.validationError(report, source, userId, chart); + if (error != null) { + return CommonResponse.failure(error); + } + try { + return CommonResponse.success(chartRefresh.execute(report, source, chart)); + } + catch (SQLException | RuntimeException e) { + LOG.warn("Chart refresh failed for report {} ({})", report.getId(), e.getClass().getSimpleName()); + return CommonResponse.failure("Unable to refresh the chart. Check the data source and SQL, then try again."); + } } @Override @Transactional public CommonResponse deleteByCode(BaseRepository repository, String code) { - repository.findByCode(code).ifPresent(value -> linkRepository.deleteReport(value.getId())); - return ReportService.super.deleteByCode(repository, code); + Optional report = repository.findByCode(code); + report.ifPresent(value -> linkRepository.deleteReport(value.getId())); + CommonResponse response = ReportService.super.deleteByCode(repository, code); + report.filter(value -> Boolean.TRUE.equals(response.getStatus())) + .ifPresent(value -> notifications.publish(NotificationScene.REPORT_DELETED, String.format("报表「%s」已删除", value.getName()), + String.format("报表「%s」已删除,它的分享链接随之失效。", value.getName()), EntityType.REPORT, value.getCode(), value.getName())); + return response; } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java new file mode 100644 index 0000000000..6c49883de6 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java @@ -0,0 +1,405 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.adhoc.Adhoc; +import org.devlive.metaforge.service.entity.DataSetEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.enums.ReportType; +import org.devlive.metaforge.service.event.NotificationEventPublisher; +import org.devlive.metaforge.service.repository.DataSetRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DataSetService; +import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +@Slf4j +@Service +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") +public class ReportShareServiceImpl + implements ReportShareService +{ + static final Pattern TOKEN = Pattern.compile("^[A-Za-z0-9_-]{32}$"); + private static final Pattern ORIGIN = Pattern.compile("^https?://(\\*\\.)?[a-z0-9-]+(\\.[a-z0-9-]+)*(:\\d{1,5})?$"); + static final int MAX_ORIGINS = 20; + // Public traffic must not turn into one database query per page view. + private static final long CACHE_MILLIS = 30_000; + private static final int CACHE_SIZE = 1000; + static final int MAX_TABLE_ROWS = 10_000; + + private final ReportRepository repository; + private final UserRepository userRepository; + private final DataSetRepository dataSetRepository; + private final ReportService reportService; + private final DataSetService dataSetService; + private final ObjectMapper mapper; + private final SecureRandom random = new SecureRandom(); + private final Map cache = new ConcurrentHashMap<>(); + + public ReportShareServiceImpl(ReportRepository repository, UserRepository userRepository, DataSetRepository dataSetRepository, + ReportService reportService, DataSetService dataSetService, ObjectMapper mapper) + { + this.repository = repository; + this.userRepository = userRepository; + this.dataSetRepository = dataSetRepository; + this.reportService = reportService; + this.dataSetService = dataSetService; + this.mapper = mapper; + } + + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + + @Override + public CommonResponse share(String code, ShareRequest request) + { + Long userId = UserDetailsService.getUser().getId(); + ReportEntity report = userId == null ? null : repository.findByCode(code) + .filter(value -> Objects.equals(value.getUserId(), userId)) + .orElse(null); + if (report == null) { + return CommonResponse.failure("Report not found"); + } + if (request.isEnabled() && !shareable(report)) { + return CommonResponse.failure("This report was saved by the earlier chart editor. Save it again from the query page to share it."); + } + if (request.isEnabled() && request.getExpiresAt() != null && request.getExpiresAt() <= System.currentTimeMillis()) { + return CommonResponse.failure("The expiry time must be in the future"); + } + Optional origins = normalizeOrigins(request.getOrigins()); + if (origins.isEmpty()) { + return CommonResponse.failure("Allowed sites must be origins such as https://example.com, at most " + MAX_ORIGINS); + } + boolean wasShared = report.isShareEnabled(); + if (request.isRotate() || report.getShareToken() == null) { + report.setShareToken(newToken()); + } + report.setShareEnabled(request.isEnabled()); + report.setShareEmbed(request.isEmbed()); + report.setShareOrigins(origins.get().isEmpty() ? null : origins.get()); + report.setShareExpiresAt(request.getExpiresAt() == null ? null : new Date(request.getExpiresAt())); + report.setShareShowTitle(!Boolean.FALSE.equals(request.getShowTitle())); + report.setShareShowDescription(!Boolean.FALSE.equals(request.getShowDescription())); + report.setShareAllowData(Boolean.TRUE.equals(request.getAllowData())); + report.setShareAllowDownload(Boolean.TRUE.equals(request.getAllowDownload())); + repository.save(report); + evict(report.getShareToken()); + if (wasShared != request.isEnabled()) { + notifications.publish(request.isEnabled() ? NotificationScene.REPORT_SHARED : NotificationScene.REPORT_UNSHARED, + String.format(request.isEnabled() ? "报表「%s」已开启分享" : "报表「%s」已关闭分享", report.getName()), + String.format(request.isEnabled() ? "报表「%s」现在可以通过分享链接访问。" : "报表「%s」的分享链接已失效。", report.getName()), + EntityType.REPORT, report.getCode(), report.getName()); + } + return CommonResponse.success(state(report)); + } + + @Override + public CommonResponse view(String token) + { + return shared(token) + .map(report -> CommonResponse.success(new PublicReport(report.getName(), + report.isShareShowDescription() ? report.getDescription() : null, + report.getType() == null ? null : report.getType().name(), report.isRealtime(), report.getConfigure(), report.isShareEmbed(), + report.isShareShowTitle(), report.isShareShowDescription(), report.isShareAllowData(), report.isShareAllowDownload()))) + .orElseGet(() -> CommonResponse.failure("Report not found")); + } + + @Override + public CommonResponse>> refresh(String token) + { + Optional report = shared(token).filter(value -> value.getType() == ReportType.QUERY && value.isRealtime()); + if (report.isEmpty()) { + return CommonResponse.failure("Dynamic chart not found"); + } + // The owner's own refresh path: read-only transaction, row limit and SQL checks included. + return cached("refresh:" + token, () -> publicly(() -> asOwner(report.get(), () -> reportService.refreshChart(report.get().getCode())))); + } + + @Override + public CommonResponse data(String token) + { + Optional report = shared(token).filter(value -> value.getType() == ReportType.DATASET); + if (report.isEmpty() || report.get().getDatasetId() == null) { + return CommonResponse.failure("Report not found"); + } + Optional dataset; + try { + dataset = dataSetRepository.findById(report.get().getDatasetId()); + } + catch (RuntimeException e) { + log.warn("Shared report dataset lookup failed: {}", e.getClass().getSimpleName()); + return CommonResponse.failure("Report not found"); + } + if (dataset.isEmpty()) { + return CommonResponse.failure("Report not found"); + } + Adhoc query; + try { + query = mapper.readValue(report.get().getQuery(), Adhoc.class); + } + catch (JsonProcessingException e) { + return CommonResponse.failure("Report query is invalid"); + } + return cached("data:" + token, () -> publicly(() -> { + CommonResponse response = asOwner(report.get(), () -> dataSetService.adhoc(dataset.get().getCode(), query)); + return Boolean.TRUE.equals(response.getStatus()) ? CommonResponse.success(response.getData()) : CommonResponse.failure(String.valueOf(response.getMessage())); + })); + } + + @Override + public CommonResponse
table(String token) + { + Optional report = shared(token).filter(ReportEntity::isShareAllowData); + if (report.isEmpty()) { + return CommonResponse.failure("Report data is not shared"); + } + return rows(report.get(), token); + } + + @Override + public Optional> download(String token) + { + Optional report = shared(token).filter(ReportEntity::isShareAllowDownload); + if (report.isEmpty()) { + return Optional.empty(); + } + CommonResponse
table = rows(report.get(), token); + return Boolean.TRUE.equals(table.getStatus()) + ? Optional.of(Map.entry(StringUtils.defaultIfBlank(report.get().getName(), "report"), table.getData())) + : Optional.empty(); + } + + /** The same rows the public chart shows: the refreshed or saved chart rows, or the dataset result. */ + private CommonResponse
rows(ReportEntity report, String token) + { + if (report.getType() == ReportType.DATASET) { + CommonResponse data = data(token); + if (!Boolean.TRUE.equals(data.getStatus())) { + return CommonResponse.failure(String.valueOf(data.getMessage())); + } + JsonNode result = mapper.valueToTree(data.getData()); + List headers = new ArrayList<>(); + result.path("headers").forEach(header -> headers.add(header.asText())); + List items = new ArrayList<>(); + result.path("columns").forEach(items::add); + return CommonResponse.success(table(headers, items)); + } + List items = new ArrayList<>(); + if (report.isRealtime()) { + CommonResponse>> refreshed = refresh(token); + if (Boolean.TRUE.equals(refreshed.getStatus())) { + refreshed.getData().forEach(row -> items.add(mapper.valueToTree(row))); + } + } + if (items.isEmpty()) { + try { + mapper.readTree(report.getConfigure()).path("columns").forEach(items::add); + } + catch (JsonProcessingException e) { + return CommonResponse.failure("Report data is unavailable"); + } + } + Set headers = new LinkedHashSet<>(); + items.stream().filter(JsonNode::isObject).forEach(row -> row.fieldNames().forEachRemaining(headers::add)); + return CommonResponse.success(table(new ArrayList<>(headers), items)); + } + + /** Rows as lists in header order; object rows by name, array rows by position. */ + static Table table(List headers, List items) + { + List> rows = new ArrayList<>(); + for (JsonNode item : items) { + if (rows.size() >= MAX_TABLE_ROWS) { + return new Table(headers, rows, true); + } + List row = new ArrayList<>(headers.size()); + for (int index = 0; index < headers.size(); index++) { + JsonNode cell = item.isObject() ? item.get(headers.get(index)) : item.isArray() ? item.get(index) : (index == 0 ? item : null); + row.add(cell == null || cell.isNull() ? null : cell.isNumber() ? cell.numberValue() : cell.isBoolean() ? cell.booleanValue() : cell.isValueNode() ? cell.asText() : cell.toString()); + } + rows.add(row); + } + return new Table(headers, rows, false); + } + + @Override + public Optional frameAncestors(String token) + { + return shared(token) + .filter(ReportEntity::isShareEmbed) + .map(report -> StringUtils.isBlank(report.getShareOrigins()) ? "*" + : "'self' " + String.join(" ", report.getShareOrigins().split("\n"))); + } + + Optional shared(String token) + { + if (token == null || !TOKEN.matcher(token).matches()) { + return Optional.empty(); + } + try { + return repository.findByShareToken(token) + .filter(report -> report.isActive() && report.isShareEnabled() && token.equals(report.getShareToken()) + && !expired(report, System.currentTimeMillis()) && shareable(report)); + } + catch (RuntimeException e) { + // Anonymous callers never see internal errors such as SQL text; to them the report is unavailable. + log.warn("Shared report lookup failed: {}", e.getClass().getSimpleName()); + return Optional.empty(); + } + } + + /** Public reads report a generic failure instead of internal error messages. */ + private static CommonResponse publicly(Supplier> action) + { + try { + return action.get(); + } + catch (RuntimeException e) { + log.warn("Shared report read failed: {}", e.getClass().getSimpleName()); + return CommonResponse.failure("Report data is temporarily unavailable"); + } + } + + static boolean expired(ReportEntity report, long now) + { + return report.getShareExpiresAt() != null && report.getShareExpiresAt().getTime() <= now; + } + + /** Dataset reports and query charts render from saved data; earlier query reports ran SQL in the browser. */ + boolean shareable(ReportEntity report) + { + if (report.getType() == ReportType.DATASET) { + return true; + } + try { + JsonNode configure = report.getConfigure() == null ? null : mapper.readTree(report.getConfigure()); + return configure != null && configure.path("queryChart").isObject(); + } + catch (JsonProcessingException e) { + return false; + } + } + + /** One origin per line, lower case and de-duplicated; empty when the input is invalid. */ + static Optional normalizeOrigins(String value) + { + Set origins = new LinkedHashSet<>(); + for (String item : StringUtils.defaultString(value).split("[\\s,]+")) { + String origin = StringUtils.removeEnd(item.trim().toLowerCase(Locale.ROOT), "/"); + if (origin.isEmpty()) { + continue; + } + if (!ORIGIN.matcher(origin).matches()) { + return Optional.empty(); + } + origins.add(origin); + } + return origins.size() > MAX_ORIGINS ? Optional.empty() : Optional.of(String.join("\n", origins)); + } + + private ShareState state(ReportEntity report) + { + return new ShareState(report.isShareEnabled(), report.isShareEmbed(), StringUtils.defaultString(report.getShareOrigins()), report.getShareToken(), + report.getShareExpiresAt() == null ? null : report.getShareExpiresAt().getTime(), report.isShareShowTitle(), report.isShareShowDescription(), + report.isShareAllowData(), report.isShareAllowDownload()); + } + + private String newToken() + { + byte[] bytes = new byte[24]; + random.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + /** Runs the saved definition with the owner's identity, so ownership checks apply as for the owner. */ + private T asOwner(ReportEntity report, Supplier action) + { + UserEntity owner = userRepository.findById(report.getUserId()).orElse(null); + SecurityContext previous = SecurityContextHolder.getContext(); + SecurityContext context = SecurityContextHolder.createEmptyContext(); + if (owner != null) { + UserDetailsService principal = new UserDetailsService(owner.getId(), owner.getCode(), owner.getUsername(), "", new ArrayList<>(), null); + context.setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); + } + SecurityContextHolder.setContext(context); + try { + return action.get(); + } + finally { + SecurityContextHolder.setContext(previous); + } + } + + @SuppressWarnings("unchecked") + private CommonResponse cached(String key, Supplier> loader) + { + long now = System.currentTimeMillis(); + Cached hit = cache.get(key); + if (hit != null && now - hit.at < CACHE_MILLIS) { + return (CommonResponse) hit.response; + } + CommonResponse response = loader.get(); + if (Boolean.TRUE.equals(response.getStatus())) { + if (cache.size() >= CACHE_SIZE) { + cache.entrySet().removeIf(entry -> now - entry.getValue().at >= CACHE_MILLIS); + if (cache.size() >= CACHE_SIZE) { + cache.clear(); + } + } + cache.put(key, new Cached(response, now)); + } + return response; + } + + private void evict(String token) + { + cache.keySet().removeIf(key -> key.endsWith(":" + token)); + } + + private static final class Cached + { + private final CommonResponse response; + private final long at; + + private Cached(CommonResponse response, long at) + { + this.response = response; + this.at = at; + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java index d8448351c8..afe5d7850f 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java @@ -25,6 +25,9 @@ import org.devlive.metaforge.service.entity.ScheduledHistoryEntity; import org.devlive.metaforge.service.entity.SourceEntity; import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; import org.devlive.metaforge.service.persistence.Persistence; import org.devlive.metaforge.service.repository.BaseRepository; import org.devlive.metaforge.service.repository.PluginAuditRepository; @@ -37,6 +40,7 @@ import org.devlive.metaforge.spi.PluginService; import org.devlive.metaforge.spi.model.Response; import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -79,6 +83,14 @@ public SourceServiceImpl( this.environment = environment; } + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + @Override public CommonResponse> getAll(BaseRepository repository, FilterBody filter) { @@ -92,8 +104,6 @@ public CommonResponse> getAll(BaseRepository delete(Long id) this.pluginAuditRepository.deleteAllBySourceId(id); this.scheduledHistoryRepository.deleteAllBySourceId(id); this.sourceRepository.deleteById(id); + notifications.publish(NotificationScene.SOURCE_DELETED, String.format("数据源「%s」已删除", source.getName()), + String.format("数据源「%s」已删除。", source.getName()), EntityType.SOURCE, source.getCode(), source.getName()); } return CommonResponse.success(id); } + @Override + public CommonResponse deleteByCode(BaseRepository repository, String code) + { + // The page deletes by code; go through delete(id) so owned rows and files go too. + return this.sourceRepository.findByCode(code) + .map(value -> { + delete(value.getId()); + return CommonResponse.success(code); + }) + .orElseGet(() -> CommonResponse.failure(String.format("Resource [ %s ] not found", code))); + } + @Override public CommonResponse getByCode(BaseRepository repository, String code) { @@ -267,6 +291,7 @@ public CommonResponse saveOrUpdate(BaseRepository saveOrUpdate(BaseRepository fsTypes() + { + return pluginManager.getPluginInfos().stream() + .filter(plugin -> plugin.getType() == PluginType.FILESYSTEM) + .map(PluginMetadata::getName) + .distinct() + .sorted() + .collect(Collectors.toList()); + } + + private FsSettings keepSecret(FsSettings fs) + { + if (StringUtils.isNotEmpty(fs.getSecret())) { + return fs; + } + return fs.toBuilder().secret(fs().getSecret()).build(); + } + + @Override + public AuthSettings auth() + { + return AuthSettings.from(configures.getEffective(SystemSettingSchema.CATEGORY, SystemSettingSchema.AUTH, + SystemSettingSchema.auth(Boolean.TRUE.equals(initializer.getRegistrationEnable()), Boolean.TRUE.equals(initializer.getCaptchaEnable())))); + } + + @Override + public MailSettings mail() + { + return MailSettings.from(configures.getEffective(SystemSettingSchema.CATEGORY, SystemSettingSchema.MAIL, SystemSettingSchema.mail())); + } + + @Override + public OAuthSettings oauth(OAuthProvider provider) + { + return OAuthSettings.from(provider, configures.getEffective(SystemSettingSchema.CATEGORY, provider.configureName(), SystemSettingSchema.oauth(provider))); + } + + @Override + public Map settings() + { + Map result = new LinkedHashMap<>(); + result.put("auth", auth()); + result.put("mail", mail().masked()); + Map oauth = new LinkedHashMap<>(); + for (OAuthProvider provider : OAuthProvider.values()) { + oauth.put(provider.getId(), oauth(provider).masked()); + } + result.put("oauth", oauth); + result.put("fs", fs().masked()); + result.put("fsTypes", fsTypes()); + return result; + } + + @Override + public CommonResponse> save(AuthSettings auth, MailSettings mail, Map oauth, FsSettings fs) + { + if (auth == null || mail == null) { + return CommonResponse.failure("The sign-in and mail settings are required"); + } + if (StringUtils.isNotBlank(auth.getSiteUrl()) && !SystemSettingSchema.isUrl(auth.getSiteUrl())) { + return CommonResponse.failure("The site address must start with http:// or https://"); + } + // Checked as a whole before anything is written, so a refused provider saves nothing. + Map providers = new EnumMap<>(OAuthProvider.class); + if (oauth != null) { + for (Map.Entry entry : oauth.entrySet()) { + OAuthProvider provider = OAuthProvider.of(entry.getKey()).orElse(null); + if (provider == null || entry.getValue() == null) { + continue; + } + OAuthSettings value = keepSecret(provider, entry.getValue()); + if (value.isEnabled() && !value.configured()) { + return CommonResponse.failure(provider.configurable() + ? "Set the client ID, the client secret and the three endpoints of " + provider.getLabel() + " before turning it on" + : "Set the client ID and the client secret of " + provider.getLabel() + " before turning it on"); + } + providers.put(provider, value); + } + } + FsSettings storage = fs == null ? null : keepSecret(fs); + if (storage != null) { + if (!fsTypes().contains(storage.getType())) { + return CommonResponse.failure("The file system plugin " + storage.getType() + " is not installed"); + } + if (!storage.configured()) { + return CommonResponse.failure("Set the access key, the secret key, the endpoint and the bucket of " + storage.getType()); + } + } + MailSettings saved = keepPassword(mail); + CommonResponse authSaved = configures.save(SystemSettingSchema.CATEGORY, SystemSettingSchema.AUTH, auth.toMap(), "Sign-in and registration"); + if (!Boolean.TRUE.equals(authSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(authSaved.getMessage())); + } + CommonResponse mailSaved = configures.save(SystemSettingSchema.CATEGORY, SystemSettingSchema.MAIL, saved.toMap(), "Mail server"); + if (!Boolean.TRUE.equals(mailSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(mailSaved.getMessage())); + } + for (Map.Entry entry : providers.entrySet()) { + OAuthProvider provider = entry.getKey(); + CommonResponse providerSaved = configures.save(SystemSettingSchema.CATEGORY, provider.configureName(), entry.getValue().toMap(provider), + provider.getLabel() + " sign-in"); + if (!Boolean.TRUE.equals(providerSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(providerSaved.getMessage())); + } + } + if (storage != null) { + CommonResponse fsSaved = configures.save(SystemSettingSchema.CATEGORY, SystemSettingSchema.FS, storage.toMap(), "File storage"); + fs = null; + if (!Boolean.TRUE.equals(fsSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(fsSaved.getMessage())); + } + } + return CommonResponse.success(settings()); + } + + @Override + public CommonResponse testFs(FsSettings fs) + { + FsSettings storage = fs == null ? fs() : keepSecret(fs); + if (!storage.configured()) { + return CommonResponse.failure("Set the access key, the secret key, the endpoint and the bucket of " + storage.getType()); + } + Optional plugin = pluginManager.getPlugin(storage.getType()); + if (plugin.isEmpty()) { + return CommonResponse.failure("The file system plugin " + storage.getType() + " is not installed"); + } + FsService service = plugin.get().getService(FsService.class); + String endpoint = storage.local() ? String.join(File.separator, initializer.getDataHome(), "test") : storage.getEndpoint(); + String file = String.join("/", "metaforge-test", UUID.randomUUID() + ".txt"); + try (InputStream probe = new ByteArrayInputStream("MetaForge storage test".getBytes(StandardCharsets.UTF_8))) { + FsResponse written = service.writer(FsRequest.builder().access(storage.getAccess()).secret(storage.getSecret()).endpoint(endpoint) + .bucket(storage.getBucket()).stream(probe).fileName(file).build()); + if (written == null || !written.isSuccessful()) { + return CommonResponse.failure("Writing a test file failed: " + (written == null ? "no response" : written.getMessage())); + } + try { + service.delete(FsRequest.builder().access(storage.getAccess()).secret(storage.getSecret()).endpoint(endpoint) + .bucket(storage.getBucket()).fileName(file).build()); + } + catch (RuntimeException e) { + // The write is what was tested; a store that can not delete keeps the small probe file. + log.warn("Removing the storage test file {} failed", file, e); + } + return CommonResponse.success(Boolean.TRUE); + } + catch (IOException | RuntimeException e) { + log.warn("Storage test with {} failed", storage.getType(), e); + return CommonResponse.failure("Writing a test file failed: " + e.getMessage()); + } + } + + @Override + public CommonResponse testRuntime(String category, String name, Map values) + { + Map given = values == null ? Map.of() : values; + if (RuntimeConfigureService.CATEGORY_EXECUTOR.equals(category)) { + Optional executor = pluginManager.getPluginInfos().stream() + .filter(meta -> meta.getType() == PluginType.EXECUTOR && meta.getName().equals(name) && meta.getInstance() instanceof Plugin) + .map(meta -> (Plugin) meta.getInstance()) + .findFirst(); + if (executor.isEmpty()) { + return CommonResponse.failure("The executor " + name + " is not installed"); + } + List problems = executor.get().testRuntimeConfigures(given); + return problems.isEmpty() ? CommonResponse.success(name) : CommonResponse.failure(String.join("; ", problems)); + } + if (RuntimeConfigureService.CATEGORY_DATASET.equals(category)) { + String type = given.getOrDefault("type", ""); + Optional target = pluginManager.getPlugin(type); + if (target.isEmpty()) { + return CommonResponse.failure("The dataset target plugin " + type + " is not installed"); + } + try { + PluginService service = target.get().getService(PluginService.class); + SourceEntity entity = SourceEntity.builder().host(given.get("host")).port(Integer.valueOf(given.getOrDefault("port", "0").trim())) + .username(given.get("username")).password(given.get("password")).database(given.get("database")).build(); + Response response = service.execute(entity.toConfigure(pluginManager, target.get()), service.validator()); + if (response == null || !Boolean.TRUE.equals(response.getIsSuccessful())) { + return CommonResponse.failure("Connecting to the dataset target failed: " + (response == null ? "no response" : response.getMessage())); + } + return CommonResponse.success(type); + } + catch (RuntimeException e) { + return CommonResponse.failure("Connecting to the dataset target failed: " + e.getMessage()); + } + } + return CommonResponse.failure("Unknown runtime settings " + category); + } + + @Override + public CommonResponse testMail(MailSettings mail, String to) + { + MailSettings settings = mail == null ? mail() : keepPassword(mail); + if (!settings.ready()) { + return CommonResponse.failure("Set the mail server and the sender address first"); + } + if (!AuthServiceImpl.isEmail(to)) { + return CommonResponse.failure("Enter a valid email address"); + } + try { + mailService.send(settings, to.trim(), "MetaForge 测试邮件 / Test mail", + "这是一封来自 MetaForge 的测试邮件,邮件服务配置可用。\nThis is a test mail from MetaForge; the mail server works."); + return CommonResponse.success(Boolean.TRUE); + } + catch (MessagingException | RuntimeException e) { + log.warn("Test mail to {} failed", to, e); + return CommonResponse.failure("Failed to send mail: " + e.getMessage()); + } + } + + private OAuthSettings keepSecret(OAuthProvider provider, OAuthSettings settings) + { + // Rebuilt from the stored form, so the fixed endpoints of GitHub and Google can not be changed. + OAuthSettings value = OAuthSettings.from(provider, settings.toMap(provider)); + if (StringUtils.isEmpty(value.getClientSecret())) { + value.setClientSecret(oauth(provider).getClientSecret()); + } + return value; + } + + private MailSettings keepPassword(MailSettings mail) + { + if (StringUtils.isNotEmpty(mail.getPassword())) { + return mail; + } + return mail.toBuilder().password(mail().getPassword()).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableCompareServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableCompareServiceImpl.java index 5fd59dcaff..f721f0cc68 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableCompareServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableCompareServiceImpl.java @@ -6,6 +6,7 @@ import org.devlive.metaforge.plugin.Plugin; import org.devlive.metaforge.plugin.PluginManager; import org.devlive.metaforge.service.body.TableCompareBody; +import org.devlive.metaforge.service.common.SourceColumns; import org.devlive.metaforge.service.entity.SourceEntity; import org.devlive.metaforge.service.repository.SourceRepository; import org.devlive.metaforge.service.service.TableCompareService; @@ -158,19 +159,16 @@ private List> structureDiff(Source source, TableCompareBody return StructureDiff.compare(columns(source, body.getSourceDatabase(), body.getSourceTable()), columns(source, body.getTargetDatabase(), body.getTargetTable())); } + /** The columns every plugin lists through getColumns, so any source with column:read can be compared. */ private static List columns(Source source, String database, String table) { - PluginService service = source.service; - Response response = service.execute(source.raw, "SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, COLUMN_COMMENT FROM information_schema.COLUMNS" - + " WHERE TABLE_SCHEMA = " + service.sqlLiteral(database) + " AND TABLE_NAME = " + service.sqlLiteral(table) + " ORDER BY ORDINAL_POSITION"); - List> rows = rows(response); - if (rows.isEmpty()) { + List columns = SourceColumns.of(source.service, source.raw, database, table).stream() + .map(column -> new StructureDiff.Column(column.getName(), column.getType(), column.isNullable(), column.getDefaultValue(), column.getComment())) + .collect(Collectors.toList()); + if (columns.isEmpty()) { throw new IllegalArgumentException("Table " + database + "." + table + " has no columns or does not exist"); } - return rows.stream() - .map(row -> new StructureDiff.Column(text(row.get(0)), text(row.get(1)), "YES".equalsIgnoreCase(text(row.get(2))), row.get(3) == null ? null : text(row.get(3)), - text(row.get(4)))) - .collect(Collectors.toList()); + return columns; } /** ADD, MODIFY and DROP COLUMN statements built from the source table's own column definitions. */ diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableExportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableExportServiceImpl.java index 6b83b6bb36..c5ec2e3706 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableExportServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableExportServiceImpl.java @@ -1,6 +1,5 @@ package org.devlive.metaforge.service.service.impl; -import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -11,6 +10,7 @@ import org.devlive.metaforge.convert.model.ConvertResponse; import org.devlive.metaforge.plugin.PluginManager; import org.devlive.metaforge.service.body.TableExportBody; +import org.devlive.metaforge.service.common.SourceColumns; import org.devlive.metaforge.service.entity.SourceEntity; import org.devlive.metaforge.service.repository.SourceRepository; import org.devlive.metaforge.service.service.TableExportService; @@ -188,18 +188,7 @@ private static long count(PluginService service, RuntimeConfigure configure, Str private static List types(PluginService service, RuntimeConfigure configure, String database, String table, List headers) { Map byName = new HashMap<>(); - Response response = service.execute(configure, "SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = " - + service.sqlLiteral(database) + " AND TABLE_NAME = " + service.sqlLiteral(table)); - if (response != null && Boolean.TRUE.equals(response.getIsSuccessful()) && response.getColumns() != null) { - for (Object row : response.getColumns()) { - if (row instanceof List && ((List) row).size() >= 2) { - byName.put(String.valueOf(((List) row).get(0)), String.valueOf(((List) row).get(1)).toUpperCase(Locale.ROOT)); - } - else if (row instanceof JsonNode) { - byName.put(((JsonNode) row).path("COLUMN_NAME").asText(), ((JsonNode) row).path("COLUMN_TYPE").asText().toUpperCase(Locale.ROOT)); - } - } - } + SourceColumns.of(service, configure, database, table).forEach(column -> byName.put(column.getName(), column.getType().toUpperCase(Locale.ROOT))); List types = new ArrayList<>(); headers.forEach(header -> types.add(byName.getOrDefault(header, ""))); return types; diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableImportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableImportServiceImpl.java index aafabbd205..83e9da4c03 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableImportServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/TableImportServiceImpl.java @@ -13,7 +13,11 @@ import org.devlive.metaforge.plugin.PluginManager; import org.devlive.metaforge.plugin.PluginMetadata; import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.service.common.SourceColumns; import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; import org.devlive.metaforge.service.importer.ImportPlan; import org.devlive.metaforge.service.importer.ImportStatements; import org.devlive.metaforge.service.importer.ImportTable; @@ -25,6 +29,7 @@ import org.devlive.metaforge.spi.PluginService; import org.devlive.metaforge.spi.model.Response; import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -38,6 +43,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -86,6 +92,14 @@ public TableImportServiceImpl(PluginManager pluginManager, SourceRepository repo this.repository = repository; } + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; + } + /** One analysed upload. */ private static final class Session { @@ -347,7 +361,21 @@ private static final class Target { private PluginService service; private RuntimeConfigure configure; - private Map>> columns; + private String database; + /** table -> [name, type, nullable, default, extra] per column, read through the plugin when a table is first checked. */ + private final Map>> columns = new HashMap<>(); + + /** The table's columns; null when it does not exist. */ + List> columns(String table) + { + List> found = columns.computeIfAbsent(table, name -> { + List> rows = new ArrayList<>(); + SourceColumns.of(service, configure, database, name).forEach(column -> rows.add(Arrays.asList(column.getName(), column.getType(), + column.isNullable() ? "YES" : "NO", column.getDefaultValue(), column.isAutoIncrement() ? "auto_increment" : ""))); + return rows; + }); + return found.isEmpty() ? null : found; + } } private Optional session(ImportPlan plan, String code, String database) @@ -385,17 +413,7 @@ private CommonResponse target(String code, String database, ImportPlan p return CommonResponse.failure("Source does not support " + missing.get()); } target.configure = entity.get().toConfigure("NoneConvert", pluginManager, plugin.get()); - Response response = target.service.execute(target.configure, "SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA" - + " FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = " + target.service.sqlLiteral(database) + " ORDER BY TABLE_NAME, ORDINAL_POSITION"); - target.columns = new HashMap<>(); - if (response != null && Boolean.TRUE.equals(response.getIsSuccessful()) && response.getColumns() != null) { - for (Object row : response.getColumns()) { - if (row instanceof List) { - List values = (List) row; - target.columns.computeIfAbsent(String.valueOf(values.get(0)), key -> new ArrayList<>()).add(new ArrayList<>(values.subList(1, values.size()))); - } - } - } + target.database = database; return CommonResponse.success(target); } @@ -477,7 +495,7 @@ private Map check(Session session, String database, ImportPlan p private static void validate(ImportTable source, ImportPlan.TablePlan table, Target target, List> messages) { String name = table.getTarget() == null ? "" : table.getTarget().trim(); - List> columns = target.columns.get(name); + List> columns = name.isEmpty() ? null : target.columns(name); if (name.isEmpty()) { messages.add(message("error", "targetMissing", "")); return; @@ -664,9 +682,22 @@ private void run(Session session, String database, ImportPlan plan, Target targe boolean failed = progress.stream().anyMatch(item -> "failed".equals(item.get("status"))); job.put("status", failed ? "failed" : "done"); job.put("finishedAt", System.currentTimeMillis()); + notifyFinished(session, database, failed, progress); } } + private void notifyFinished(Session session, String database, boolean failed, List> progress) + { + long done = progress.stream().filter(item -> "done".equals(item.get("status"))).count(); + String reason = progress.stream().filter(item -> "failed".equals(item.get("status"))) + .map(item -> item.get("target") + ": " + item.get("message")).findFirst().orElse(""); + notifications.publishTo(session.user, failed ? NotificationScene.IMPORT_FAILED : NotificationScene.IMPORT_SUCCEEDED, + String.format(failed ? "导入到 %s 失败" : "导入到 %s 完成", database), + failed ? String.format("%d/%d 张表导入成功。%s", done, progress.size(), reason) + : String.format("%d 张表已导入 %s。", done, database), + EntityType.SOURCE, session.code, database); + } + private static void importTable(Session session, String database, ImportPlan plan, Target target, ImportStatements statements, ImportPlan.TablePlan table, Map item) { diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UploadServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UploadServiceImpl.java index c55b6a761d..58ce1826cc 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UploadServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UploadServiceImpl.java @@ -14,6 +14,7 @@ import org.devlive.metaforge.service.enums.UploadMode; import org.devlive.metaforge.service.initializer.InitializerConfigure; import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.SystemSettingService; import org.devlive.metaforge.service.service.UploadService; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -28,12 +29,14 @@ public class UploadServiceImpl implements UploadService { + private final SystemSettingService systemSettings; private final PluginManager pluginManager; private final HttpServletRequest request; private final InitializerConfigure initializer; - public UploadServiceImpl(PluginManager pluginManager, HttpServletRequest request, InitializerConfigure initializer) + public UploadServiceImpl(PluginManager pluginManager, HttpServletRequest request, InitializerConfigure initializer, SystemSettingService systemSettings) { + this.systemSettings = systemSettings; this.pluginManager = pluginManager; this.request = request; this.initializer = initializer; @@ -45,17 +48,17 @@ public CommonResponse upload(UploadBody configure) if (configure.getMode().equals(UploadMode.DASHBOARD)) { configure.setCode(CodeUtils.generateCode(false)); AvatarEntity entity = AvatarEntity.builder() - .type(initializer.getFsConfigure().getType()) + .type(systemSettings.fs().getType()) .build(); try { FsRequest fsRequest = getFsRequest(configure.getFile(), configure); - pluginManager.getPlugin(initializer.getFsConfigure().getType()) + pluginManager.getPlugin(systemSettings.fs().getType()) .ifPresent(plugin -> { FsService fsService = plugin.getService(FsService.class); FsResponse response = fsService.writer(fsRequest); entity.setPath(response.getRemote()); entity.setLocal(response.getRemote()); - if (initializer.getFsConfigure().getType().startsWith("Local")) { + if (systemSettings.fs().getType().startsWith("Local")) { entity.setPath(getAccess(entity)); } }); @@ -89,10 +92,10 @@ private FsRequest getFsRequest(MultipartFile file, UploadBody configure) throws IOException { return FsRequest.builder() - .access(initializer.getFsConfigure().getAccess()) - .secret(initializer.getFsConfigure().getSecret()) + .access(systemSettings.fs().getAccess()) + .secret(systemSettings.fs().getSecret()) .endpoint(getHome(configure)) - .bucket(initializer.getFsConfigure().getBucket()) + .bucket(systemSettings.fs().getBucket()) .stream(file.getInputStream()) .fileName("avatar.png") .build(); @@ -106,8 +109,8 @@ private FsRequest getFsRequest(MultipartFile file, UploadBody configure) */ private String getHome(UploadBody configure) { - if (!initializer.getFsConfigure().getType().startsWith("Local")) { - return initializer.getFsConfigure().getEndpoint(); + if (!systemSettings.fs().getType().startsWith("Local")) { + return systemSettings.fs().getEndpoint(); } return String.join("/", initializer.getDataHome(), UserDetailsService.getUser().getUsername(), diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java index 40ae5b8ec7..34962f079e 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java @@ -27,6 +27,9 @@ import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.entity.convert.AvatarEntity; import org.devlive.metaforge.service.entity.itransient.user.UserEditorEntity; +import org.devlive.metaforge.service.enums.EntityType; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.event.NotificationEventPublisher; import org.devlive.metaforge.service.initializer.InitializerConfigure; import org.devlive.metaforge.service.model.AiModel; import org.devlive.metaforge.service.persistence.Persistence; @@ -41,8 +44,11 @@ import org.devlive.metaforge.service.repository.jdbc.LinkRepository; import org.devlive.metaforge.service.repository.jdbc.MenuAccessRepository; import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.security.UserDetailsServiceImpl; import org.devlive.metaforge.service.service.JwtService; +import org.devlive.metaforge.service.service.SystemSettingService; import org.devlive.metaforge.service.service.UserService; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.dao.DataAccessException; import org.springframework.data.domain.Page; @@ -57,6 +63,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -81,6 +88,7 @@ public class UserServiceImpl implements UserService { + private final SystemSettingService systemSettings; private final UserRepository userRepository; private final RoleRepository roleRepository; private final SourceRepository sourceRepository; @@ -97,9 +105,11 @@ public class UserServiceImpl private final InitializerConfigure initializer; private final PluginManager pluginManager; private final HttpServletRequest request; + private final UserDetailsServiceImpl userDetailsService; - public UserServiceImpl(UserRepository userRepository, RoleRepository roleRepository, SourceRepository sourceRepository, MenuAccessRepository menuRepository, LinkRepository linkRepository, NotificationRepository notificationRepository, PluginAuditRepository pluginAuditRepository, ScheduledHistoryRepository scheduledHistoryRepository, PasswordEncoder encoder, AuthenticationManager authenticationManager, JwtService jwtService, RedisTemplate redisTemplate, Environment environment, InitializerConfigure initializer, PluginManager pluginManager, HttpServletRequest request) + public UserServiceImpl(UserRepository userRepository, RoleRepository roleRepository, SourceRepository sourceRepository, MenuAccessRepository menuRepository, LinkRepository linkRepository, NotificationRepository notificationRepository, PluginAuditRepository pluginAuditRepository, ScheduledHistoryRepository scheduledHistoryRepository, PasswordEncoder encoder, AuthenticationManager authenticationManager, JwtService jwtService, RedisTemplate redisTemplate, Environment environment, InitializerConfigure initializer, PluginManager pluginManager, HttpServletRequest request, UserDetailsServiceImpl userDetailsService, SystemSettingService systemSettings) { + this.systemSettings = systemSettings; this.userRepository = userRepository; this.linkRepository = linkRepository; this.notificationRepository = notificationRepository; @@ -116,6 +126,15 @@ public UserServiceImpl(UserRepository userRepository, RoleRepository roleReposit this.initializer = initializer; this.pluginManager = pluginManager; this.request = request; + this.userDetailsService = userDetailsService; + } + + private NotificationEventPublisher notifications = NotificationEventPublisher.NONE; + + @Autowired + public void setNotifications(NotificationEventPublisher notifications) + { + this.notifications = notifications; } @Override @@ -208,13 +227,24 @@ public CommonResponse saveOrUpdate(BaseRepository @AuditUserLog @Override - public CommonResponse authenticate(UserEntity configure) + public CommonResponse authenticate(UserEntity configure, long expiration) { UsernamePasswordAuthenticationToken authenticationToken = new UsernamePasswordAuthenticationToken(configure.getUsername(), configure.getPassword()); - Authentication authentication = authenticationManager.authenticate(authenticationToken); + return token(authenticationManager.authenticate(authenticationToken), expiration); + } + + @AuditUserLog + @Override + public CommonResponse issue(String username, long expiration) + { + UserDetails details = userDetailsService.loadUserByUsername(username); + return token(new UsernamePasswordAuthenticationToken(details, null, details.getAuthorities()), expiration); + } + private CommonResponse token(Authentication authentication, long expiration) + { SecurityContextHolder.getContext().setAuthentication(authentication); - String jwt = jwtService.generateJwtToken(authentication); + String jwt = jwtService.generateJwtToken(authentication, expiration); UserDetailsService userDetails = (UserDetailsService) authentication.getPrincipal(); List roles = userDetails.getAuthorities().stream() @@ -258,6 +288,8 @@ public CommonResponse resetPassword(String code, UserResetPasswordBody con .map(user -> { user.setPassword(encoder.encode(configure.getPassword())); userRepository.save(user); + notifications.publishTo(user.getCode(), NotificationScene.PASSWORD_CHANGED, "登录密码已被重置", + "管理员重置了你的登录密码;如非预期,请联系管理员。", EntityType.USER, user.getCode(), user.getUsername()); return CommonResponse.success(user.getId()); }) .orElseGet(() -> CommonResponse.failure(ServiceState.USER_NOT_FOUND)); @@ -291,6 +323,8 @@ public CommonResponse changePassword(UserPasswordBody configure) } user.setPassword(encoder.encode(configure.getNewPassword())); this.userRepository.save(user); + notifications.publish(NotificationScene.PASSWORD_CHANGED, "登录密码已修改", "你的登录密码已修改;如非本人操作,请立即重置密码。", + EntityType.USER, user.getCode(), user.getUsername()); return CommonResponse.success(user.getId()); } @@ -431,17 +465,17 @@ public CommonResponse changeEditorConfigure(UserEditorEntity configu public CommonResponse uploadAvatar(UploadBody configure) { AvatarEntity entity = AvatarEntity.builder() - .type(initializer.getFsConfigure().getType()) + .type(systemSettings.fs().getType()) .build(); try { FsRequest fsRequest = getFsRequest(configure.getFile(), configure); - pluginManager.getPlugin(initializer.getFsConfigure().getType()) + pluginManager.getPlugin(systemSettings.fs().getType()) .ifPresent(plugin -> { FsService fsService = plugin.getService(FsService.class); FsResponse response = fsService.writer(fsRequest); entity.setPath(response.getRemote()); entity.setLocal(response.getRemote()); - if (initializer.getFsConfigure().getType().startsWith("Local")) { + if (systemSettings.fs().getType().startsWith("Local")) { entity.setPath(getAccess(entity)); } @@ -469,25 +503,14 @@ public CommonResponse uploadAvatar(UploadBody configure) } } - @Override - public CommonResponse changeNotify(UserEntity configure) - { - return userRepository.findByCode(UserDetailsService.getUser().getCode()) - .map(value -> { - value.setNotifyConfigure(configure.getNotifyConfigure()); - return CommonResponse.success(userRepository.save(value)); - }) - .orElseGet(() -> CommonResponse.failure(ServiceState.USER_NOT_FOUND)); - } - private FsRequest getFsRequest(MultipartFile file, UploadBody configure) throws IOException { return FsRequest.builder() - .access(initializer.getFsConfigure().getAccess()) - .secret(initializer.getFsConfigure().getSecret()) + .access(systemSettings.fs().getAccess()) + .secret(systemSettings.fs().getSecret()) .endpoint(getHome(configure)) - .bucket(initializer.getFsConfigure().getBucket()) + .bucket(systemSettings.fs().getBucket()) .stream(file.getInputStream()) .fileName("avatar.png") .build(); @@ -495,8 +518,8 @@ private FsRequest getFsRequest(MultipartFile file, UploadBody configure) private String getHome(UploadBody configure) { - if (!initializer.getFsConfigure().getType().startsWith("Local")) { - return initializer.getFsConfigure().getEndpoint(); + if (!systemSettings.fs().getType().startsWith("Local")) { + return systemSettings.fs().getEndpoint(); } return String.join( "/", diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/WorkflowServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/WorkflowServiceImpl.java index 4333afe2de..cc0a79010e 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/WorkflowServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/WorkflowServiceImpl.java @@ -1,45 +1,64 @@ package org.devlive.metaforge.service.service.impl; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import com.fasterxml.jackson.databind.ObjectMapper; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.devlive.metaforge.common.response.CommonResponse; -import org.devlive.metaforge.common.utils.NullAwareBeanUtils; +import org.devlive.metaforge.common.utils.CodeUtils; import org.devlive.metaforge.executor.ExecutorService; import org.devlive.metaforge.executor.common.RunState; import org.devlive.metaforge.executor.configure.ExecutorConfigure; import org.devlive.metaforge.executor.configure.ExecutorRequest; import org.devlive.metaforge.executor.configure.ExecutorResponse; +import org.devlive.metaforge.executor.workflow.WorkflowPlan; +import org.devlive.metaforge.plugin.Plugin; import org.devlive.metaforge.plugin.PluginManager; -import org.devlive.metaforge.service.common.FolderUtils; +import org.devlive.metaforge.service.entity.SourceEntity; import org.devlive.metaforge.service.entity.UserEntity; import org.devlive.metaforge.service.entity.WorkflowEntity; import org.devlive.metaforge.service.initializer.InitializerConfigure; -import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; +import org.devlive.metaforge.service.itransient.configuration.Configuration; import org.devlive.metaforge.service.repository.BaseRepository; +import org.devlive.metaforge.service.repository.SourceRepository; import org.devlive.metaforge.service.repository.WorkflowRepository; import org.devlive.metaforge.service.security.UserDetailsService; import org.devlive.metaforge.service.service.RuntimeConfigureService; import org.devlive.metaforge.service.service.WorkflowService; -import org.springframework.core.env.Environment; +import org.devlive.metaforge.service.workflow.WorkflowCatalog; +import org.devlive.metaforge.service.workflow.WorkflowExecutorDefinition; +import org.devlive.metaforge.service.workflow.WorkflowRun; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.adapter.RowCallback; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import java.io.File; -import java.io.FileInputStream; +import javax.annotation.PreDestroy; + import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.Timestamp; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Optional; -import java.util.concurrent.Executors; - -import static com.google.common.base.Preconditions.checkArgument; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service @@ -47,268 +66,655 @@ public class WorkflowServiceImpl implements WorkflowService { + /** Rows returned by a query preview in the editor. */ + private static final int PREVIEW_ROWS = 50; private final InitializerConfigure initializer; - private final PluginManager pluginManager; - private final Environment environment; + private final PluginManager plugins; private final WorkflowRepository repository; - private final RuntimeConfigureService runtimeConfigureService; - - public WorkflowServiceImpl(InitializerConfigure initializer, PluginManager pluginManager, Environment environment, WorkflowRepository repository, RuntimeConfigureService runtimeConfigureService) + private final SourceRepository sources; + private final RuntimeConfigureService runtime; + private final WorkflowCatalog catalog; + private final ObjectMapper mapper = new ObjectMapper(); + private final Map active = new ConcurrentHashMap<>(); + private final ThreadPoolExecutor workers = new ThreadPoolExecutor(2, 4, 60, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(32), runnable -> { + Thread thread = new Thread(runnable, "workflow-worker"); + thread.setDaemon(true); + return thread; + }); + + public WorkflowServiceImpl(InitializerConfigure initializer, PluginManager plugins, WorkflowRepository repository, + SourceRepository sources, RuntimeConfigureService runtime, WorkflowCatalog catalog) { this.initializer = initializer; - this.pluginManager = pluginManager; - this.environment = environment; + this.plugins = plugins; this.repository = repository; - this.runtimeConfigureService = runtimeConfigureService; + this.sources = sources; + this.runtime = runtime; + this.catalog = catalog; + } + + @Override + public synchronized CommonResponse saveOrUpdate(BaseRepository ignored, WorkflowEntity form) + { + UserEntity user = UserDetailsService.getUser(); + WorkflowEntity entity = new WorkflowEntity(); + if (form.getCode() != null) { + Optional previous = repository.findByCode(form.getCode()); + if (previous.isPresent()) { + if (!owned(previous.get(), user)) { + return CommonResponse.failure("Workflow not found"); + } + entity = previous.get(); + } + else if (form.getId() != null) { + return CommonResponse.failure("Workflow not found"); + } + } + if (active.containsKey(form.getCode() == null ? "" : form.getCode()) || entity.getState() == RunState.RUNNING) { + return CommonResponse.failure("Stop the workflow before editing it"); + } + if (form.getName() == null || form.getName().isBlank() || form.getName().length() > 200) { + return CommonResponse.failure("Workflow name is required and must not exceed 200 characters"); + } + try { + catalog.executor(form.getExecutor()); + entity.setCode(form.getCode() == null ? CodeUtils.generateCode(false) : form.getCode()); + entity.setName(form.getName().trim()); + entity.setExecutor(form.getExecutor()); + if (form.getConfigure() != null) { + Integer revision = Optional.ofNullable(entity.getConfigure()).map(Configuration::getRevision).orElse(0); + form.getConfigure().setRevision(revision); + form.getConfigure().setPublishedAt(null); + } + entity.setConfigure(form.getConfigure()); + entity.setUserId(user.getId()); + entity.setWork(workHome(user, entity)); + if (entity.getState() == null) { + entity.setState(RunState.CREATED); + } + return CommonResponse.success(repository.save(entity)); + } + catch (IllegalArgumentException e) { + return CommonResponse.failure(e.getMessage()); + } + } + + @Override + public CommonResponse getByCode(BaseRepository ignored, String code) + { + return own(code).map(CommonResponse::success).orElseGet(() -> CommonResponse.failure("Workflow not found")); + } + + @Override + public CommonResponse getById(BaseRepository ignored, Long id) + { + return repository.findById(id).filter(entity -> owned(entity, UserDetailsService.getUser())) + .map(CommonResponse::success).orElseGet(() -> CommonResponse.failure("Workflow not found")); } @Override - public CommonResponse saveOrUpdate(BaseRepository repository, WorkflowEntity configure) + public CommonResponse> validate(WorkflowEntity workflow) { - log.info("Starting workflow save/update process for executor: {}, code: {}", - configure.getExecutor(), configure.getCode()); - repository.findByCode(configure.getCode()) - .ifPresent(value -> NullAwareBeanUtils.copyNullProperties(value, configure)); + return CommonResponse.success(catalog.validate(workflow)); + } - if (configure.getState() != null && configure.getState().equals(RunState.RUNNING)) { + @Override + public synchronized CommonResponse publish(String code) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + if (active.containsKey(code)) { + return CommonResponse.failure("Stop the workflow before publishing"); + } + List errors = catalog.validate(entity); + if (!errors.isEmpty()) { + return CommonResponse.failure(String.join("\n", errors)); + } + entity.getConfigure().setPublishedAt(System.currentTimeMillis()); + entity.getConfigure().setRevision(Optional.ofNullable(entity.getConfigure().getRevision()).orElse(0) + 1); + return CommonResponse.success(repository.save(entity)); + } + + @Override + public CommonResponse> catalog() + { + try { + return CommonResponse.success(catalog.catalog()); + } + catch (IllegalArgumentException e) { + return CommonResponse.failure(e.getMessage()); + } + } + + @Override + public synchronized CommonResponse restart(String code) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + if (active.containsKey(code) || entity.getState() == RunState.RUNNING) { return CommonResponse.failure("Workflow is already running"); } + List errors = catalog.validate(entity); + if (!errors.isEmpty()) { + return CommonResponse.failure(String.join("\n", errors)); + } + try { + UserEntity user = UserDetailsService.getUser(); + WorkflowCatalog.WorkflowExecutor executor = catalog.executor(entity.getExecutor()); + Plugin plugin = executor.plugin(); + WorkflowRun run = new WorkflowRun(); + run.setId(UUID.randomUUID().toString()); + run.setCode(code); + run.setName(entity.getName()); + run.setExecutor(entity.getExecutor()); + run.setState(RunState.RUNNING); + run.setStartedAt(System.currentTimeMillis()); + run.setNodes(entity.getConfigure().getNodes().stream().map(node -> Map.of( + "id", node.getId(), "label", node.getLabel() == null ? node.getTid() : node.getLabel(), + "category", node.getCategory())).collect(Collectors.toList())); + entity.setWork(workHome(user, entity)); + Map options = new HashMap<>(runtime.getEffective(RuntimeConfigureService.CATEGORY_EXECUTOR, plugin.getName(), plugin.runtimeConfigures())); + WorkflowPlan plan = catalog.plan(entity, run.getId(), user.getUsername(), options, sourceCode -> source(sourceCode, user)); + ExecutorRequest request = executor.service().compile(plan); + request.setPluginManager(plugins); + request.setMetricsListener(metrics -> run.setMetrics(metrics)); + Files.createDirectories(Path.of(entity.getWork())); + writeRun(entity, run); + entity.setState(RunState.RUNNING); + entity.setMessage(null); + entity.setElapsed(null); + repository.save(entity); + ActiveRun handle = new ActiveRun(run, request, executor.service(), executor.definition().getCapabilities()); + active.put(code, handle); + try { + handle.future = workers.submit(() -> execute(entity, handle)); + } + catch (RuntimeException e) { + active.remove(code); + finish(entity, handle, RunState.FAILURE, e.getMessage()); + throw e; + } + return CommonResponse.success(run.getId()); + } + catch (IOException | RuntimeException e) { + log.warn("Cannot start workflow {}", code, e); + return CommonResponse.failure(e.getMessage()); + } + } + + private void execute(WorkflowEntity entity, ActiveRun handle) + { + try { + synchronized (this) { + if (handle.run.getState() != RunState.RUNNING) { + return; + } + handle.started = true; + } + ExecutorResponse response = handle.service.start(handle.request); + synchronized (this) { + handle.run.setMetrics(response.getMetrics()); + // Executors that do not report row counts leave the count unknown rather than zero. + boolean hasCount = handle.capabilities.isRowCount() || response.getMetrics() != null || response.getCount() > 0; + handle.run.setCount(hasCount ? response.getCount() : null); + handle.run.setPreview(response.getPreview()); + if (handle.run.getState() == RunState.RUNNING) { + finish(entity, handle, response.getState(), response.getMessage()); + } + else { + writeRun(entity, handle.run); + } + } + } + catch (Exception e) { + synchronized (this) { + if (handle.run.getState() == RunState.RUNNING) { + finish(entity, handle, RunState.FAILURE, e.getMessage()); + } + } + log.warn("Workflow {} failed", entity.getCode(), e); + } + finally { + active.remove(entity.getCode(), handle); + } + } - UserEntity user = UserDetailsService.getUser(); - String workHome = FolderUtils.getWorkHome( - initializer.getDataHome(), - user.getUsername(), - String.join(File.separator, "workflow", configure.getExecutor().toLowerCase(), configure.getCode()) - ); - log.debug("Created work directory: {}", workHome); - - configure.setState(RunState.CREATED); - configure.setUser(user); - configure.setWork(workHome); - configure.setFrom(null); - configure.setTo(null); - WorkflowEntity saved = repository.save(configure); - log.info("Workflow initialized with state CREATED, code: {}", saved.getCode()); - - configure.setState(RunState.RUNNING); - saved = repository.save(configure); - log.info("Workflow updated to RUNNING state, code: {}", saved.getCode()); - - pluginManager.getPlugin(configure.getExecutor()) - .ifPresentOrElse( - plugin -> { - log.info("Found plugin for executor: {}", configure.getExecutor()); - final java.util.concurrent.ExecutorService executorService = Executors.newSingleThreadExecutor(); + private void finish(WorkflowEntity entity, ActiveRun handle, RunState state, String message) + { + handle.run.setMessage(message); + handle.run.setFinishedAt(System.currentTimeMillis()); + handle.run.setElapsed(handle.run.getFinishedAt() - handle.run.getStartedAt()); + handle.run.setState(state == null ? RunState.FAILURE : state); + entity.setState(handle.run.getState()); + entity.setMessage(message); + entity.setElapsed(handle.run.getElapsed()); + entity.setUpdateTime(new Date()); + repository.save(entity); + writeRun(entity, handle.run); + } + + @Override + public synchronized CommonResponse stop(String code) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + ActiveRun handle = active.get(code); + if (handle == null) { + return CommonResponse.failure("Workflow is not running on this server"); + } + if (!handle.capabilities.stoppable()) { + return CommonResponse.failure("This executor cannot stop a running workflow"); + } + try { + boolean cooperative = handle.started && handle.capabilities.cooperativeStop(); + if (cooperative) { + ExecutorResponse response = handle.service.stop(handle.request); + if (!response.getSuccessful()) { + return CommonResponse.failure(response.getMessage()); + } + } + if (handle.future != null && !cooperative) { + handle.future.cancel(true); + } + finish(entity, handle, RunState.STOPPED, "Stopped by user"); + // Keep the handle registered until the interrupted execution has actually exited. + if (!handle.started) { + active.remove(code, handle); + } + return CommonResponse.success(true); + } + catch (Exception e) { + return CommonResponse.failure(e.getMessage()); + } + } + + @Override + public CommonResponse>> queryColumns(String source, String query) + { + CommonResponse> read = readQuery(source, query, 0); + if (!Boolean.TRUE.equals(read.getStatus())) { + return CommonResponse.failure(String.valueOf(read.getMessage())); + } + @SuppressWarnings("unchecked") + List> columns = (List>) read.getData().get("columns"); + return CommonResponse.success(columns); + } + + @Override + public CommonResponse> queryPreview(String source, String query) + { + return readQuery(source, query, PREVIEW_ROWS); + } + + /** + * Reads the columns and at most {@code limit} rows of a query through the data source plugin's + * streaming read, stopping as soon as enough is known; the query itself is never rewritten. + */ + private CommonResponse> readQuery(String source, String query, int limit) + { + String statement = query == null ? "" : query.strip().replaceAll("[;\\s]+$", ""); + if (source == null || source.isBlank() || statement.isEmpty()) { + return CommonResponse.failure("Select a data source and enter a query"); + } + try { + ExecutorConfigure configure = source(source, UserDetailsService.getUser()); + PluginService plugin = configure.getPlugin(); + if (plugin == null || !plugin.supportsStreaming()) { + return CommonResponse.failure("The data source cannot describe query columns"); + } + List> columns = new ArrayList<>(); + List> rows = new ArrayList<>(); + List names = new ArrayList<>(); + long started = System.currentTimeMillis(); + try { + plugin.executeStream(configure.getOriginConfigure(), statement, Math.max(1, limit), new RowCallback() + { + @Override + public void onStatement(java.sql.Statement jdbc) + { + try { + jdbc.setMaxRows(Math.max(1, limit)); + } + catch (java.sql.SQLException ignored) { + // Not every driver supports a row limit; reading stops after the limit either way. + } + } + + @Override + public void onSchema(List headers, List types) + { + for (int index = 0; index < headers.size(); index++) { + Map column = new LinkedHashMap<>(); + column.put("name", headers.get(index)); + column.put("type", types != null && index < types.size() && types.get(index) != null ? types.get(index) : ""); + columns.add(column); + names.add(headers.get(index)); + } + if (limit <= 0) { + throw new SchemaRead(); + } + } + + @Override + public void onRow(List row) + { + if (rows.size() >= limit) { + throw new SchemaRead(); + } + Map values = new LinkedHashMap<>(); + for (int index = 0; index < names.size(); index++) { + Object value = index < row.size() ? row.get(index) : null; + // Driver types such as timestamps or blobs are shown as text. + values.put(names.get(index), value == null || value instanceof Number || value instanceof Boolean ? value : abbreviate(value.toString())); + } + rows.add(values); + if (rows.size() >= limit) { + throw new SchemaRead(); + } + } + }); + } + catch (RuntimeException stopped) { + // Expected once enough is read; plugins may wrap the stop in their own exception. + if (!(stopped instanceof SchemaRead) && columns.isEmpty()) { + throw stopped; + } + } + Map result = new LinkedHashMap<>(); + result.put("columns", columns); + result.put("rows", rows); + result.put("elapsed", System.currentTimeMillis() - started); + return CommonResponse.success(result); + } + catch (RuntimeException e) { + log.warn("Cannot read a workflow query", e); + return CommonResponse.failure(e.getMessage()); + } + } + + private static String abbreviate(String value) + { + return value.length() > 1000 ? value.substring(0, 1000) + "…" : value; + } + + /** Stops a schema-only read once the plugin has reported the columns. */ + private static final class SchemaRead + extends RuntimeException + { + private static final long serialVersionUID = 1L; + + private SchemaRead() + { + super(null, null, false, false); + } + } + + /** A saved data source the user owns or that is published, with its plugin and connection configuration. */ + private ExecutorConfigure source(String code, UserEntity user) + { + SourceEntity source = sources.findByCode(code) + .filter(value -> Objects.equals(value.getUserId(), user.getId()) || Boolean.TRUE.equals(value.getPublish())) + .orElseThrow(() -> new IllegalArgumentException("Data source is unavailable")); + Plugin plugin = plugins.getPlugin(source.getType()).orElseThrow(() -> new IllegalArgumentException("Data source plugin is unavailable")); + ExecutorConfigure configure = new ExecutorConfigure(source.getType()); + configure.setPlugin(plugin.getService(PluginService.class)); + RuntimeConfigure origin = source.toConfigure("NoneConvert", plugins, plugin); + origin.setPluginManager(plugins); + configure.setOriginConfigure(origin); + return configure; + } + + @Override + public CommonResponse> runs(String code) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + Map result = new HashMap<>(); + if (entity.getWork() != null && Files.isDirectory(Path.of(entity.getWork()))) { + try (Stream files = Files.list(Path.of(entity.getWork()))) { + files.filter(file -> file.getFileName().toString().matches("run-[a-f0-9-]{36}\\.json")) + .sorted(Comparator.comparingLong((Path path) -> path.toFile().lastModified()).reversed()) + .limit(100) + .forEach(file -> { try { - Files.createDirectories(Path.of(configure.getWork())); - log.debug("Created work directories at: {}", configure.getWork()); - - Optional inputNode = configure.getConfigure() - .getNodes() - .stream() - .filter(v -> v.getCategory().equalsIgnoreCase("source")) - .findFirst(); - checkArgument(inputNode.isPresent(), "Source node must not be null"); - log.debug("Found source node: {}", inputNode.get().getKey()); - - Optional outputNode = configure.getConfigure() - .getNodes() - .stream() - .filter(v -> v.getCategory().equalsIgnoreCase("sink")) - .findFirst(); - checkArgument(outputNode.isPresent(), "Sink node must not be null"); - log.debug("Found sink node: {}", outputNode.get().getKey()); - - Optional transformNode = configure.getConfigure() - .getNodes() - .stream() - .filter(v -> v.getCategory().equalsIgnoreCase("transform")) - .findFirst(); - - ExecutorConfigure form = new ExecutorConfigure( - inputNode.get().getKey(), - MapUtils.toProperties(inputNode.get().getData()), - Sets.newHashSet() - ); - - ExecutorConfigure transform = null; - if (transformNode.isPresent()) { - transform = new ExecutorConfigure( - transformNode.get().getKey(), - MapUtils.toProperties(transformNode.get().getData()), - Sets.newHashSet() - ); + WorkflowRun run = mapper.readValue(file.toFile(), WorkflowRun.class); + if (run.getState() == RunState.RUNNING && !active.containsKey(code)) { + run.setState(RunState.INTERRUPTED); + run.setMessage("Execution interrupted by server restart"); } - - ExecutorConfigure to = new ExecutorConfigure( - outputNode.get().getKey(), - MapUtils.toProperties(outputNode.get().getData()), - Sets.newHashSet() - ); - - // 从 metaforge_configure 读 effective 配置 - java.util.Map executorCfg = runtimeConfigureService.getEffective( - RuntimeConfigureService.CATEGORY_EXECUTOR, - plugin.getName(), - plugin.runtimeConfigures() - ); - log.debug("Executor home directory: {}", executorCfg.get("home")); - - ExecutorRequest request = new ExecutorRequest( - configure.getCode(), - user.getUsername(), - form, - to - ); - request.setWorkHome(configure.getWork()); - request.setTransform(transform); - // executor 专属项(home/startScript/way/mode/engine 等)整体作为 options 传入 - request.setOptions(executorCfg); - log.info("Created executor request for workflow: {}", configure.getCode()); - - Timestamp start = new Timestamp(System.currentTimeMillis()); - executorService.submit(() -> { - try { - log.info("Starting async execution for workflow: {}", configure.getCode()); - ExecutorService pluginService = plugin.getService(ExecutorService.class); - - repository.findByCode(configure.getCode()) - .ifPresent(finalConfigure -> { - log.debug("Found workflow by code: {}", configure.getCode()); - initializer.getTaskExecutors().put(finalConfigure.getCode(), executorService); - - log.info("Executing workflow: {}", finalConfigure.getCode()); - ExecutorResponse response = pluginService.start(request); - log.info("Execution completed with state: {}", response.getState()); - - finalConfigure.setUpdateTime(new Timestamp(System.currentTimeMillis())); - finalConfigure.setState(response.getState()); - finalConfigure.setMessage(response.getMessage()); - finalConfigure.setElapsed(finalConfigure.getUpdateTime().getTime() - start.getTime()); - - WorkflowEntity updated = repository.save(finalConfigure); - log.info("Workflow updated with final state: {}, elapsed time: {} ms", - updated.getState(), updated.getElapsed()); - }); - } - catch (Exception e) { - log.error("Error during workflow execution: {}", configure.getCode(), e); - } - finally { - log.info("Cleaning up resources for workflow: {}", configure.getCode()); - initializer.getTaskExecutors().remove(configure.getCode()); - executorService.shutdownNow(); - } - }); + result.put(run.getId(), run); } catch (IOException e) { - log.error("Failed to create work directory for workflow: {}", configure.getCode(), e); - configure.setState(RunState.FAILURE); - configure.setMessage(ExceptionUtils.getStackTrace(e)); - repository.save(configure); - throw new RuntimeException(e); + log.warn("Cannot read workflow run {}", file, e); } - }, - () -> { - String errorMsg = String.format("Executor [ %s ] not found", configure.getExecutor()); - log.error(errorMsg); - configure.setState(RunState.FAILURE); - configure.setMessage(errorMsg); - repository.save(configure); - throw new RuntimeException(errorMsg); - } - ); + }); + } + catch (IOException e) { + return CommonResponse.failure("Cannot read execution records"); + } + } + ActiveRun running = active.get(code); + if (running != null) { + result.put(running.run.getId(), running.run); + } + return CommonResponse.success(result.values().stream().sorted(Comparator.comparingLong(WorkflowRun::getStartedAt).reversed()).limit(100).map(this::summary).collect(Collectors.toList())); + } - log.info("Workflow process initiated successfully for: {}", configure.getCode()); - return CommonResponse.success(configure); + private WorkflowRun summary(WorkflowRun run) + { + WorkflowRun summary = new WorkflowRun(); + summary.setId(run.getId()); + summary.setCode(run.getCode()); + summary.setName(run.getName()); + summary.setExecutor(run.getExecutor()); + summary.setStartedAt(run.getStartedAt()); + summary.setFinishedAt(run.getFinishedAt()); + summary.setElapsed(run.getElapsed()); + summary.setState(run.getState()); + summary.setCount(run.getCount()); + return summary; } @Override - public CommonResponse stop(String code) + public CommonResponse run(String code, String runId) { - return (CommonResponse) repository.findByCode(code) - .map(entity -> { - if (entity.getState().equals(RunState.STOPPED) - || entity.getState().equals(RunState.FAILURE) - || entity.getState().equals(RunState.SUCCESS) - || entity.getState().equals(RunState.TIMEOUT)) { - return CommonResponse.failure(String.format("Workflow [ %s ] is already stopped", entity.getName())); - } - - java.util.concurrent.ExecutorService service = initializer.getTaskExecutors() - .get(entity.getName()); - if (service != null) { - service.shutdownNow(); - } - entity.setState(RunState.STOPPED); - entity.setMessage(null); - this.repository.save(entity); - return CommonResponse.success(true); - }) - .orElseGet(() -> CommonResponse.failure(String.format("Workflow [ %s ] not found", code))); + WorkflowEntity entity = own(code).orElse(null); + if (entity == null || runId == null || !runId.matches("[a-f0-9-]{36}")) { + return CommonResponse.failure("Execution not found"); + } + ActiveRun running = active.get(code); + if (running != null && running.run.getId().equals(runId)) { + return CommonResponse.success(running.run); + } + if (entity.getWork() != null) { + Path path = Path.of(entity.getWork(), "run-" + runId + ".json"); + try { + WorkflowRun run = mapper.readValue(path.toFile(), WorkflowRun.class); + if (run.getState() == RunState.RUNNING) { + run.setState(RunState.INTERRUPTED); + run.setMessage("Execution interrupted by server restart"); + } + return CommonResponse.success(run); + } + catch (IOException e) { + return CommonResponse.failure("Execution not found"); + } + } + return CommonResponse.failure("Execution not found"); } @Override public CommonResponse> log(String code) { - return repository.findByCode(code) - .map(entity -> { - if (entity.getState().equals(RunState.QUEUE) - || entity.getState().equals(RunState.CREATED)) { - return CommonResponse.>failure(String.format("Workflow [ %s ] is not running", entity.getName())); - } + return log(code, null); + } - List lines = Lists.newArrayList(); - try (FileInputStream stream = new FileInputStream(String.format("%s/%s.log", entity.getWork(), entity.getCode()))) { - lines.addAll(IOUtils.readLines(stream, "UTF-8")); - return CommonResponse.success(lines); - } - catch (IOException e) { - log.error("Failed to read pipeline [ {} ] log ", entity.getName(), e); - return CommonResponse.>failure("Failed to read log file"); - } - }) - .orElse(CommonResponse.failure(String.format("Workflow [ %s ] not found", code))); + @Override + public CommonResponse> log(String code, String runId) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + if (runId != null && !runId.matches("[a-f0-9-]{36}")) { + return CommonResponse.failure("Invalid run identifier"); + } + if (runId == null) { + List records = runs(code).getData(); + runId = records == null || records.isEmpty() ? code : records.get(0).getId(); + } + if (entity.getWork() == null || !Files.exists(Path.of(entity.getWork(), runId + ".log"))) { + return CommonResponse.success(List.of()); + } + Path path = Path.of(entity.getWork(), runId + ".log"); + try (var file = new java.io.RandomAccessFile(path.toFile(), "r")) { + // Read only the log tail; polling a long-running job must not load an unbounded file. + long offset = Math.max(0, file.length() - 262144); + file.seek(offset); + byte[] bytes = new byte[(int) (file.length() - offset)]; + file.readFully(bytes); + List lines = new ArrayList<>(List.of(new String(bytes, StandardCharsets.UTF_8).split("\n"))); + if (offset > 0 && !lines.isEmpty()) { + lines.remove(0); + } + return CommonResponse.success(lines); + } + catch (IOException e) { + return CommonResponse.failure("Cannot read execution log"); + } } @Override - public CommonResponse deleteByCode(BaseRepository repository, String code) + public CommonResponse> statistics() { - return repository.findByCode(code) - .map(entity -> { - if (entity.getState().equals(RunState.RUNNING)) { - return CommonResponse.failure(String.format("Workflow [ %s ] is already running", entity.getName())); - } + List rows = repository.findAllByUserId(UserDetailsService.getUser().getId()); + Map counts = new LinkedHashMap<>(); + counts.put("total", (long) rows.size()); + counts.put("running", rows.stream().filter(row -> row.getState() == RunState.RUNNING).count()); + counts.put("success", rows.stream().filter(row -> row.getState() == RunState.SUCCESS).count()); + counts.put("failure", rows.stream().filter(row -> row.getState() == RunState.FAILURE || row.getState() == RunState.TIMEOUT || row.getState() == RunState.INTERRUPTED).count()); + return CommonResponse.success(counts); + } - repository.delete(entity); - try { - FileUtils.deleteDirectory(new File(entity.getWork())); - } - catch (IOException e) { - log.warn("Failed to delete work directory", e); - } - return CommonResponse.success(code); - }) - .orElseGet(() -> CommonResponse.failure(String.format("Resource [ %s ] not found", code))); + @Override + public synchronized CommonResponse deleteByCode(BaseRepository ignored, String code) + { + WorkflowEntity entity = own(code).orElse(null); + if (entity == null) { + return CommonResponse.failure("Workflow not found"); + } + if (active.containsKey(code) || entity.getState() == RunState.RUNNING) { + return CommonResponse.failure("Stop the workflow before deleting it"); + } + repository.delete(entity); + if (entity.getWork() != null) { + try { + FileUtils.deleteDirectory(Path.of(entity.getWork()).toFile()); + } + catch (IOException e) { + log.warn("Cannot delete workflow work directory", e); + } + } + return CommonResponse.success(code); } @Override - public CommonResponse restart(String code) + public CommonResponse deleteById(BaseRepository ignored, Long id) { - return repository.findByCode(code) + return repository.findById(id).filter(entity -> owned(entity, UserDetailsService.getUser())) .map(entity -> { - if (entity.getState().equals(RunState.RUNNING)) { - return CommonResponse.failure(String.format("Workflow [ %s ] is already running", entity.getName())); - } + CommonResponse response = deleteByCode(repository, entity.getCode()); + return Boolean.TRUE.equals(response.getStatus()) ? CommonResponse.success(id) : CommonResponse.failure(String.valueOf(response.getMessage())); + }).orElseGet(() -> CommonResponse.failure("Workflow not found")); + } - CommonResponse startResponse = saveOrUpdate(repository, entity); - if (startResponse.getStatus()) { - return CommonResponse.success(code); - } - else { - return CommonResponse.failure(startResponse.getMessage().toString()); - } - }) - .orElse(CommonResponse.failure(String.format("Workflow [ %s ] not found", code))); + private Optional own(String code) + { + UserEntity user = UserDetailsService.getUser(); + return repository.findByCode(code).filter(entity -> owned(entity, user)); + } + + private boolean owned(WorkflowEntity entity, UserEntity user) + { + return user != null && user.getId() != null && Objects.equals(entity.getUserId(), user.getId()); + } + + private String workHome(UserEntity user, WorkflowEntity entity) + { + return entity.getWork() != null ? entity.getWork() : Path.of(initializer.getDataHome(), user.getUsername(), "workflow", entity.getCode()).toString(); + } + + private void writeRun(WorkflowEntity entity, WorkflowRun run) + { + try { + Path folder = Path.of(entity.getWork()); + Files.createDirectories(folder); + Path temporary = Files.createTempFile(folder, ".run-", ".tmp"); + try { + mapper.writeValue(temporary.toFile(), run); + Files.move(temporary, folder.resolve("run-" + run.getId() + ".json"), StandardCopyOption.REPLACE_EXISTING); + } + finally { + Files.deleteIfExists(temporary); + } + } + catch (IOException e) { + throw new IllegalStateException("Cannot store execution record", e); + } + } + + @PreDestroy + public synchronized void close() + { + for (ActiveRun handle : active.values()) { + if (handle.started && handle.capabilities.cooperativeStop()) { + handle.service.stop(handle.request); + } + } + workers.shutdownNow(); + } + + @EventListener(ApplicationReadyEvent.class) + public void recoverInterrupted() + { + try { + repository.findAllByState(RunState.RUNNING).forEach(entity -> { + entity.setState(RunState.INTERRUPTED); + entity.setMessage("Execution interrupted by server restart"); + repository.save(entity); + }); + } + catch (org.springframework.dao.DataAccessException ex) { + // 未安装的空库还没有表,安装向导完成前没有可恢复的运行;不能因此让服务退出 + // A database that is not installed yet has no tables and nothing to recover; the server must still start. + log.warn("Workflow recovery skipped: {}", ex.getMostSpecificCause().getMessage()); + } + } + + private static class ActiveRun + { + private final WorkflowRun run; + private final ExecutorRequest request; + private final ExecutorService service; + private final WorkflowExecutorDefinition.Capabilities capabilities; + private Future future; + private boolean started; + + private ActiveRun(WorkflowRun run, ExecutorRequest request, ExecutorService service, WorkflowExecutorDefinition.Capabilities capabilities) + { + this.run = run; + this.request = request; + this.service = service; + this.capabilities = capabilities; + } } } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java new file mode 100644 index 0000000000..f6e04a28a1 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java @@ -0,0 +1,70 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Sign-in and registration features an administrator turns on in the system settings. + * The email features also need a configured mail server; see {@link SystemSettingSchema}. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AuthSettings +{ + private boolean registration; + private boolean captcha; + private boolean rememberMe; + private int rememberDays; + private boolean emailSignin; + private boolean emailVerification; + private boolean passwordReset; + private String termsUrl; + private String privacyUrl; + /** Where MetaForge is opened, without a trailing slash; empty to take it from each request. */ + private String siteUrl; + + public static AuthSettings from(Map values) + { + return AuthSettings.builder() + .registration(Boolean.parseBoolean(values.get("registration"))) + .captcha(Boolean.parseBoolean(values.get("captcha"))) + .rememberMe(Boolean.parseBoolean(values.get("rememberMe"))) + .rememberDays(SystemSettingSchema.number(values.get("rememberDays"), 30, 1, 365)) + .emailSignin(Boolean.parseBoolean(values.get("emailSignin"))) + .emailVerification(Boolean.parseBoolean(values.get("emailVerification"))) + .passwordReset(Boolean.parseBoolean(values.get("passwordReset"))) + .termsUrl(SystemSettingSchema.text(values.get("termsUrl"))) + .privacyUrl(SystemSettingSchema.text(values.get("privacyUrl"))) + .siteUrl(site(values.get("siteUrl"))) + .build(); + } + + public Map toMap() + { + Map values = new LinkedHashMap<>(); + values.put("registration", String.valueOf(registration)); + values.put("captcha", String.valueOf(captcha)); + values.put("rememberMe", String.valueOf(rememberMe)); + values.put("rememberDays", String.valueOf(Math.max(1, Math.min(365, rememberDays)))); + values.put("emailSignin", String.valueOf(emailSignin)); + values.put("emailVerification", String.valueOf(emailVerification)); + values.put("passwordReset", String.valueOf(passwordReset)); + values.put("termsUrl", SystemSettingSchema.text(termsUrl)); + values.put("privacyUrl", SystemSettingSchema.text(privacyUrl)); + values.put("siteUrl", site(siteUrl)); + return values; + } + + private static String site(String value) + { + return StringUtils.stripEnd(SystemSettingSchema.text(value), "/"); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java new file mode 100644 index 0000000000..656cf303ec --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java @@ -0,0 +1,88 @@ +package org.devlive.metaforge.service.settings; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; + +import java.security.SecureRandom; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Six-digit codes mailed for sign-up and password reset. A code lasts ten minutes, a new one is + * sent at most once a minute per address, and five wrong tries discard it. + */ +@Component +public class EmailCodes +{ + public static final int MINUTES = 10; + public static final long RESEND_SECONDS = 60; + private static final int ATTEMPTS = 5; + + private final SecureRandom random = new SecureRandom(); + private final Cache codes = CacheBuilder.newBuilder() + .expireAfterWrite(MINUTES, TimeUnit.MINUTES) + .maximumSize(10000) + .build(); + + public enum Purpose + { + SIGNUP, RESET + } + + @AllArgsConstructor + private static final class Entry + { + private final String code; + private final long sent; + private int attempts; + } + + /** Seconds before another code may be sent to the address; 0 when it may be sent now. */ + public long waiting(String email, Purpose purpose) + { + Entry entry = codes.getIfPresent(key(email, purpose)); + if (entry == null) { + return 0; + } + long passed = (System.currentTimeMillis() - entry.sent) / 1000; + return Math.max(0, RESEND_SECONDS - passed); + } + + /** A new code for the address, replacing any earlier one. */ + public String issue(String email, Purpose purpose) + { + String code = String.format("%06d", random.nextInt(1_000_000)); + codes.put(key(email, purpose), new Entry(code, System.currentTimeMillis(), 0)); + return code; + } + + public void discard(String email, Purpose purpose) + { + codes.invalidate(key(email, purpose)); + } + + /** Checks a code; a matching code is used up, and too many wrong ones discard it. */ + public synchronized boolean verify(String email, Purpose purpose, String code) + { + String key = key(email, purpose); + Entry entry = codes.getIfPresent(key); + if (entry == null || code == null) { + return false; + } + if (entry.code.equals(code.trim())) { + codes.invalidate(key); + return true; + } + if (++entry.attempts >= ATTEMPTS) { + codes.invalidate(key); + } + return false; + } + + private static String key(String email, Purpose purpose) + { + return purpose.name() + ":" + email.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/FsSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/FsSettings.java new file mode 100644 index 0000000000..061576a3af --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/FsSettings.java @@ -0,0 +1,75 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Where uploads and query results are stored: one of the installed file system plugins (LocalFs, + * AliOssFs, AmazonS3Fs, MinIOFs, QiniuFs, TencentCosFs) and the account, endpoint and bucket of the + * object stores. + */ +@Data +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class FsSettings +{ + public static final String LOCAL = "LocalFs"; + + private String type; + private String access; + private String secret; + private String endpoint; + private String bucket; + /** Read only: whether a secret is stored, since the secret itself is never sent back. */ + private boolean secretSet; + + public static FsSettings from(Map values) + { + String secret = SystemSettingSchema.text(values.get("secret")); + String type = SystemSettingSchema.text(values.get("type")); + return FsSettings.builder() + .type(type.isEmpty() ? LOCAL : type) + .access(SystemSettingSchema.text(values.get("access"))) + .secret(secret) + .endpoint(SystemSettingSchema.text(values.get("endpoint"))) + .bucket(SystemSettingSchema.text(values.get("bucket"))) + .secretSet(!secret.isEmpty()) + .build(); + } + + public Map toMap() + { + Map values = new LinkedHashMap<>(); + values.put("type", StringUtils.isBlank(type) ? LOCAL : type.trim()); + values.put("access", SystemSettingSchema.text(access)); + values.put("secret", SystemSettingSchema.text(secret)); + values.put("endpoint", SystemSettingSchema.text(endpoint)); + values.put("bucket", SystemSettingSchema.text(bucket)); + return values; + } + + /** The local file system needs no account; it writes under the data directory. */ + public boolean local() + { + return StringUtils.isBlank(type) || type.startsWith("Local"); + } + + /** An object store needs its account, endpoint and bucket. */ + public boolean configured() + { + return local() || (StringUtils.isNotBlank(access) && StringUtils.isNotBlank(secret) && StringUtils.isNotBlank(endpoint) && StringUtils.isNotBlank(bucket)); + } + + /** The settings as the administrator sees them: everything except the secret. */ + public FsSettings masked() + { + return toBuilder().secret("").secretSet(StringUtils.isNotEmpty(secret)).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java new file mode 100644 index 0000000000..5017ea8e89 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java @@ -0,0 +1,10 @@ +package org.devlive.metaforge.service.settings; + +import javax.mail.MessagingException; + +/** Sends a plain text mail through the configured SMTP server. */ +public interface MailService +{ + void send(MailSettings settings, String to, String subject, String text) + throws MessagingException; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java new file mode 100644 index 0000000000..3a5f1144c2 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java @@ -0,0 +1,65 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** The SMTP server that sends verification codes. */ +@Data +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class MailSettings +{ + private String host; + private int port; + private boolean ssl; + private String username; + private String password; + private String from; + /** Read only: whether a password is stored, since the password itself is never sent back. */ + private boolean passwordSet; + + public static MailSettings from(Map values) + { + String password = SystemSettingSchema.text(values.get("password")); + return MailSettings.builder() + .host(SystemSettingSchema.text(values.get("host"))) + .port(SystemSettingSchema.number(values.get("port"), 465, 1, 65535)) + .ssl(Boolean.parseBoolean(values.get("ssl"))) + .username(SystemSettingSchema.text(values.get("username"))) + .password(password) + .from(SystemSettingSchema.text(values.get("from"))) + .passwordSet(!password.isEmpty()) + .build(); + } + + public Map toMap() + { + Map values = new LinkedHashMap<>(); + values.put("host", SystemSettingSchema.text(host)); + values.put("port", String.valueOf(Math.max(1, Math.min(65535, port)))); + values.put("ssl", String.valueOf(ssl)); + values.put("username", SystemSettingSchema.text(username)); + values.put("password", SystemSettingSchema.text(password)); + values.put("from", SystemSettingSchema.text(from)); + return values; + } + + /** Mail can be sent once a server and a sender address are set. */ + public boolean ready() + { + return StringUtils.isNotBlank(host) && StringUtils.isNotBlank(from); + } + + /** The settings as the administrator sees them: everything except the password. */ + public MailSettings masked() + { + return toBuilder().password("").passwordSet(StringUtils.isNotEmpty(password)).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthClient.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthClient.java new file mode 100644 index 0000000000..e6a6363cf7 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthClient.java @@ -0,0 +1,160 @@ +package org.devlive.metaforge.service.settings; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Exchanges an authorization code for the signed-in account. It only calls the endpoints of the + * provider's settings, never an address that came with the request. + */ +@Component +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class OAuthClient +{ + private static final int TIMEOUT = 10000; + + private final RestTemplate rest; + + @Autowired + public OAuthClient() + { + this(template()); + } + + public OAuthClient(RestTemplate rest) + { + this.rest = rest; + } + + private static RestTemplate template() + { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(TIMEOUT); + factory.setReadTimeout(TIMEOUT); + return new RestTemplate(factory); + } + + /** The account at the provider; the email is only set when the provider says it is verified. */ + @Getter + @AllArgsConstructor + public static final class Profile + { + private final String subject; + private final String email; + /** A name to derive a username from: the login, the preferred username or the name. */ + private final String login; + } + + public Profile profile(OAuthProvider provider, OAuthSettings settings, String code, String redirectUri, String verifier) + { + String token = token(settings, code, redirectUri, verifier); + Map user = get(settings.getUserinfoUrl(), token); + if (provider == OAuthProvider.GITHUB) { + Object id = user.get("id"); + if (id == null) { + throw new RestClientException("The profile has no id"); + } + return new Profile(String.valueOf(id), githubEmail(token), text(user.get("login"))); + } + String subject = text(user.get("sub")); + if (subject.isEmpty()) { + throw new RestClientException("The userinfo has no subject"); + } + String email = verified(user.get("email_verified")) ? text(user.get("email")) : ""; + String login = StringUtils.defaultString(StringUtils.firstNonBlank(text(user.get("preferred_username")), text(user.get("name")))); + return new Profile(subject, email, login); + } + + private String token(OAuthSettings settings, String code, String redirectUri, String verifier) + { + MultiValueMap form = new LinkedMultiValueMap<>(); + form.add("grant_type", "authorization_code"); + form.add("code", code); + form.add("redirect_uri", redirectUri); + form.add("client_id", settings.getClientId()); + form.add("client_secret", settings.getClientSecret()); + form.add("code_verifier", verifier); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); + @SuppressWarnings("unchecked") + Map body = rest.postForObject(settings.getTokenUrl(), new HttpEntity<>(form, headers), Map.class); + // GitHub answers a bad code with 200 and an error field. + String token = body == null ? "" : text(body.get("access_token")); + if (token.isEmpty()) { + throw new RestClientException("No access token: " + (body == null ? "empty response" : text(body.get("error")))); + } + return token; + } + + private String githubEmail(String token) + { + List emails; + try { + emails = exchange(OAuthProvider.GITHUB_EMAILS, token, List.class); + } + catch (RestClientException e) { + // Without the user:email scope the list is refused; the sign-in goes on without an email. + return ""; + } + if (emails == null) { + return ""; + } + for (Object item : emails) { + if (item instanceof Map) { + Map email = (Map) item; + if (verified(email.get("primary")) && verified(email.get("verified"))) { + return text(email.get("email")); + } + } + } + return ""; + } + + @SuppressWarnings("unchecked") + private Map get(String url, String token) + { + Map body = exchange(url, token, Map.class); + if (body == null) { + throw new RestClientException("Empty userinfo response"); + } + return body; + } + + private T exchange(String url, String token, Class type) + { + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(token); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); + // GitHub refuses requests without a user agent. + headers.set(HttpHeaders.USER_AGENT, "MetaForge"); + return rest.exchange(url, HttpMethod.GET, new HttpEntity<>(headers), type).getBody(); + } + + private static boolean verified(Object value) + { + return Boolean.TRUE.equals(value) || "true".equalsIgnoreCase(String.valueOf(value)); + } + + private static String text(Object value) + { + return value == null ? "" : String.valueOf(value).trim(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthProvider.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthProvider.java new file mode 100644 index 0000000000..f93696c756 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthProvider.java @@ -0,0 +1,62 @@ +package org.devlive.metaforge.service.settings; + +import lombok.Getter; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; + +/** + * The third-party sign-in providers. GitHub and Google have fixed endpoints; the generic OpenID + * Connect provider ("SSO") uses the endpoints an administrator configures. + */ +@Getter +public enum OAuthProvider +{ + GITHUB("github", "GitHub", "https://github.com/login/oauth/authorize", "https://github.com/login/oauth/access_token", + "https://api.github.com/user", "read:user user:email"), + GOOGLE("google", "Google", "https://accounts.google.com/o/oauth2/v2/auth", "https://oauth2.googleapis.com/token", + "https://openidconnect.googleapis.com/v1/userinfo", "openid email profile"), + OIDC("oidc", "SSO", "", "", "", "openid email profile"); + + /** GitHub leaves the email out of the profile when it is private; its verified addresses are listed here. */ + public static final String GITHUB_EMAILS = "https://api.github.com/user/emails"; + + private final String id; + private final String label; + private final String authorizationUrl; + private final String tokenUrl; + private final String userinfoUrl; + private final String scopes; + + OAuthProvider(String id, String label, String authorizationUrl, String tokenUrl, String userinfoUrl, String scopes) + { + this.id = id; + this.label = label; + this.authorizationUrl = authorizationUrl; + this.tokenUrl = tokenUrl; + this.userinfoUrl = userinfoUrl; + this.scopes = scopes; + } + + /** Whether the administrator sets the endpoints, rather than them being fixed. */ + public boolean configurable() + { + return this == OIDC; + } + + /** The name of its row in metaforge_configure under the SYSTEM category. */ + public String configureName() + { + return "OAUTH_" + name(); + } + + public static Optional of(String id) + { + if (id == null) { + return Optional.empty(); + } + String value = id.trim().toLowerCase(Locale.ROOT); + return Arrays.stream(values()).filter(provider -> provider.id.equals(value)).findFirst(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthSettings.java new file mode 100644 index 0000000000..fecfbad7bf --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthSettings.java @@ -0,0 +1,97 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * One third-party sign-in provider. It is offered on the sign-in page only once it is turned on + * and fully configured; the endpoints and scopes of GitHub and Google are fixed. + */ +@Data +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class OAuthSettings +{ + private boolean enabled; + private String clientId; + private String clientSecret; + /** Read only: whether a secret is stored, since the secret itself is never sent back. */ + private boolean secretSet; + /** The button label of the OpenID Connect provider; "SSO" when empty. */ + private String name; + private String issuer; + private String authorizationUrl; + private String tokenUrl; + private String userinfoUrl; + private String scopes; + + public static OAuthSettings from(OAuthProvider provider, Map values) + { + String secret = SystemSettingSchema.text(values.get("clientSecret")); + OAuthSettings settings = OAuthSettings.builder() + .enabled(Boolean.parseBoolean(values.get("enabled"))) + .clientId(SystemSettingSchema.text(values.get("clientId"))) + .clientSecret(secret) + .secretSet(!secret.isEmpty()) + .name("") + .issuer("") + .authorizationUrl(provider.getAuthorizationUrl()) + .tokenUrl(provider.getTokenUrl()) + .userinfoUrl(provider.getUserinfoUrl()) + .scopes(provider.getScopes()) + .build(); + if (provider.configurable()) { + settings.setName(SystemSettingSchema.text(values.get("name"))); + settings.setIssuer(SystemSettingSchema.text(values.get("issuer"))); + settings.setAuthorizationUrl(SystemSettingSchema.text(values.get("authorizationUrl"))); + settings.setTokenUrl(SystemSettingSchema.text(values.get("tokenUrl"))); + settings.setUserinfoUrl(SystemSettingSchema.text(values.get("userinfoUrl"))); + settings.setScopes(StringUtils.defaultIfBlank(SystemSettingSchema.text(values.get("scopes")), provider.getScopes())); + } + return settings; + } + + /** The stored values; the fixed endpoints of GitHub and Google are not stored. */ + public Map toMap(OAuthProvider provider) + { + Map values = new LinkedHashMap<>(); + values.put("enabled", String.valueOf(enabled)); + values.put("clientId", SystemSettingSchema.text(clientId)); + values.put("clientSecret", SystemSettingSchema.text(clientSecret)); + if (provider.configurable()) { + values.put("name", SystemSettingSchema.text(name)); + values.put("issuer", SystemSettingSchema.text(issuer)); + values.put("authorizationUrl", SystemSettingSchema.text(authorizationUrl)); + values.put("tokenUrl", SystemSettingSchema.text(tokenUrl)); + values.put("userinfoUrl", SystemSettingSchema.text(userinfoUrl)); + values.put("scopes", StringUtils.defaultIfBlank(SystemSettingSchema.text(scopes), provider.getScopes())); + } + return values; + } + + /** Everything a sign-in needs is set: the client, and for OpenID Connect its three endpoints. */ + public boolean configured() + { + return StringUtils.isNotBlank(clientId) && StringUtils.isNotBlank(clientSecret) + && SystemSettingSchema.isUrl(authorizationUrl) && SystemSettingSchema.isUrl(tokenUrl) && SystemSettingSchema.isUrl(userinfoUrl); + } + + /** Offered on the sign-in page. */ + public boolean ready() + { + return enabled && configured(); + } + + /** The settings as the administrator sees them: everything except the secret. */ + public OAuthSettings masked() + { + return toBuilder().clientSecret("").secretSet(StringUtils.isNotEmpty(clientSecret)).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthStates.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthStates.java new file mode 100644 index 0000000000..52e180216e --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/OAuthStates.java @@ -0,0 +1,96 @@ +package org.devlive.metaforge.service.settings; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * Third-party sign-ins in progress. A sign-in started at the provider must come back within ten + * minutes with its state; the ticket it ends with is exchanged for the token within a minute. + * Both are used once. + */ +@Component +public class OAuthStates +{ + private final SecureRandom random = new SecureRandom(); + private final Cache pending = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .maximumSize(10000) + .build(); + private final Cache tickets = CacheBuilder.newBuilder() + .expireAfterWrite(1, TimeUnit.MINUTES) + .maximumSize(10000) + .build(); + + @Getter + @AllArgsConstructor + public static final class Pending + { + private final String state; + private final OAuthProvider provider; + /** The PKCE code verifier sent with the code. */ + private final String verifier; + /** The page to return to after signing in, already checked to be a path on this site. */ + private final String redirect; + } + + public Pending begin(OAuthProvider provider, String redirect) + { + Pending value = new Pending(random(), provider, random(), redirect); + pending.put(value.getState(), value); + return value; + } + + public Optional take(String state) + { + if (state == null) { + return Optional.empty(); + } + return Optional.ofNullable(pending.asMap().remove(state)); + } + + /** Keeps the sign-in result until the page exchanges the returned ticket for it. */ + public String ticket(Object auth) + { + String ticket = random(); + tickets.put(ticket, auth); + return ticket; + } + + public Optional redeem(String ticket) + { + if (ticket == null) { + return Optional.empty(); + } + return Optional.ofNullable(tickets.asMap().remove(ticket)); + } + + /** The S256 code challenge of a verifier. */ + public static String challenge(String verifier) + { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(verifier.getBytes(StandardCharsets.US_ASCII)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(digest); + } + catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + private String random() + { + byte[] bytes = new byte[32]; + random.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java new file mode 100644 index 0000000000..88eae12ac0 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java @@ -0,0 +1,59 @@ +package org.devlive.metaforge.service.settings; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +@Component +public class SmtpMailService + implements MailService +{ + private static final String TIMEOUT = "10000"; + + @Override + public void send(MailSettings settings, String to, String subject, String text) + throws MessagingException + { + Properties properties = new Properties(); + properties.put("mail.transport.protocol", "smtp"); + properties.put("mail.smtp.host", settings.getHost()); + properties.put("mail.smtp.port", String.valueOf(settings.getPort())); + properties.put("mail.smtp.connectiontimeout", TIMEOUT); + properties.put("mail.smtp.timeout", TIMEOUT); + properties.put("mail.smtp.writetimeout", TIMEOUT); + if (settings.isSsl()) { + properties.put("mail.smtp.ssl.enable", "true"); + } + else { + properties.put("mail.smtp.starttls.enable", "true"); + } + boolean auth = StringUtils.isNotEmpty(settings.getUsername()); + properties.put("mail.smtp.auth", String.valueOf(auth)); + Session session = Session.getInstance(properties, auth ? new Authenticator() + { + @Override + protected PasswordAuthentication getPasswordAuthentication() + { + return new PasswordAuthentication(settings.getUsername(), settings.getPassword()); + } + } : null); + + MimeMessage message = new MimeMessage(session); + message.setFrom(new InternetAddress(settings.getFrom())); + message.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to)); + message.setSubject(subject, StandardCharsets.UTF_8.name()); + message.setText(text, StandardCharsets.UTF_8.name()); + Transport.send(message); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java new file mode 100644 index 0000000000..9df54c8515 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java @@ -0,0 +1,118 @@ +package org.devlive.metaforge.service.settings; + +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureField; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * The system settings stored in metaforge_configure under the SYSTEM category. + * Registration and captcha default to metaforge.registration.enable and metaforge.captcha.enable; + * every other feature is off until an administrator turns it on. Each third-party sign-in provider + * has a row of its own, named by {@link OAuthProvider#configureName()}. + */ +public final class SystemSettingSchema +{ + public static final String CATEGORY = "SYSTEM"; + public static final String AUTH = "AUTH"; + public static final String MAIL = "MAIL"; + public static final String FS = "FS"; + + private SystemSettingSchema() + {} + + public static List auth(boolean registration, boolean captcha) + { + return Arrays.asList( + new RuntimeConfigureField("registration", RuntimeConfigureFieldType.BOOLEAN, String.valueOf(registration), "Allow new accounts to sign up"), + new RuntimeConfigureField("captcha", RuntimeConfigureFieldType.BOOLEAN, String.valueOf(captcha), "Ask for the arithmetic captcha on sign-in and sign-up"), + new RuntimeConfigureField("rememberMe", RuntimeConfigureFieldType.BOOLEAN, "false", "Offer to keep the sign-in for several days"), + new RuntimeConfigureField("rememberDays", RuntimeConfigureFieldType.NUMBER, "30", "Days a remembered sign-in lasts"), + new RuntimeConfigureField("emailSignin", RuntimeConfigureFieldType.BOOLEAN, "false", "Allow signing in with the account email"), + new RuntimeConfigureField("emailVerification", RuntimeConfigureFieldType.BOOLEAN, "false", "Require an email and its code to sign up"), + new RuntimeConfigureField("passwordReset", RuntimeConfigureFieldType.BOOLEAN, "false", "Allow resetting a forgotten password by email"), + new RuntimeConfigureField("termsUrl", RuntimeConfigureFieldType.STRING, "", "Terms of service shown on sign-up"), + new RuntimeConfigureField("privacyUrl", RuntimeConfigureFieldType.STRING, "", "Privacy policy shown on sign-up"), + new RuntimeConfigureField("siteUrl", RuntimeConfigureFieldType.STRING, "", "Address MetaForge is opened at, for third-party sign-in callbacks")); + } + + /** File storage; the metaforge.fs.* properties are the defaults until it is saved here. */ + public static List fs(String type, String access, String secret, String endpoint, String bucket) + { + return Arrays.asList( + new RuntimeConfigureField("type", RuntimeConfigureFieldType.STRING, StringUtils.isBlank(type) ? FsSettings.LOCAL : type, "File system plugin"), + new RuntimeConfigureField("access", RuntimeConfigureFieldType.STRING, StringUtils.defaultString(access), "Access key"), + new RuntimeConfigureField("secret", RuntimeConfigureFieldType.PASSWORD, StringUtils.defaultString(secret), "Secret key"), + new RuntimeConfigureField("endpoint", RuntimeConfigureFieldType.STRING, StringUtils.defaultString(endpoint), "Endpoint or domain"), + new RuntimeConfigureField("bucket", RuntimeConfigureFieldType.STRING, StringUtils.defaultString(bucket), "Bucket")); + } + + public static List mail() + { + return Arrays.asList( + new RuntimeConfigureField("host", RuntimeConfigureFieldType.STRING, "", "SMTP host"), + new RuntimeConfigureField("port", RuntimeConfigureFieldType.NUMBER, "465", "SMTP port"), + new RuntimeConfigureField("ssl", RuntimeConfigureFieldType.BOOLEAN, "true", "SSL on connect; STARTTLS otherwise"), + new RuntimeConfigureField("username", RuntimeConfigureFieldType.STRING, "", "SMTP account"), + new RuntimeConfigureField("password", RuntimeConfigureFieldType.PASSWORD, "", "SMTP password"), + new RuntimeConfigureField("from", RuntimeConfigureFieldType.STRING, "", "Sender address")); + } + + public static List oauth(OAuthProvider provider) + { + List fields = new ArrayList<>(Arrays.asList( + new RuntimeConfigureField("enabled", RuntimeConfigureFieldType.BOOLEAN, "false", "Offer signing in with " + provider.getLabel()), + new RuntimeConfigureField("clientId", RuntimeConfigureFieldType.STRING, "", "OAuth client ID"), + new RuntimeConfigureField("clientSecret", RuntimeConfigureFieldType.PASSWORD, "", "OAuth client secret"))); + if (provider.configurable()) { + fields.addAll(Arrays.asList( + new RuntimeConfigureField("name", RuntimeConfigureFieldType.STRING, "", "Name shown for the provider"), + new RuntimeConfigureField("issuer", RuntimeConfigureFieldType.STRING, "", "Issuer"), + new RuntimeConfigureField("authorizationUrl", RuntimeConfigureFieldType.STRING, "", "Authorization endpoint"), + new RuntimeConfigureField("tokenUrl", RuntimeConfigureFieldType.STRING, "", "Token endpoint"), + new RuntimeConfigureField("userinfoUrl", RuntimeConfigureFieldType.STRING, "", "Userinfo endpoint"), + new RuntimeConfigureField("scopes", RuntimeConfigureFieldType.STRING, provider.getScopes(), "Scopes"))); + } + return fields; + } + + /** An absolute http or https address with a host. */ + public static boolean isUrl(String value) + { + if (StringUtils.isBlank(value)) { + return false; + } + try { + URI uri = new URI(value.trim()); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + return (scheme.equals("http") || scheme.equals("https")) && StringUtils.isNotBlank(uri.getHost()); + } + catch (URISyntaxException e) { + return false; + } + } + + static String text(String value) + { + return value == null ? "" : value.trim(); + } + + static int number(String value, int fallback, int min, int max) + { + if (StringUtils.isBlank(value)) { + return fallback; + } + try { + return Math.max(min, Math.min(max, (int) Double.parseDouble(value.trim()))); + } + catch (NumberFormatException e) { + return fallback; + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowCatalog.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowCatalog.java new file mode 100644 index 0000000000..988d6acd86 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowCatalog.java @@ -0,0 +1,391 @@ +package org.devlive.metaforge.service.workflow; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.devlive.metaforge.executor.ExecutorService; +import org.devlive.metaforge.executor.configure.ExecutorConfigure; +import org.devlive.metaforge.executor.workflow.WorkflowCategory; +import org.devlive.metaforge.executor.workflow.WorkflowNode; +import org.devlive.metaforge.executor.workflow.WorkflowPlan; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.service.entity.WorkflowEntity; +import org.devlive.metaforge.service.itransient.configuration.Configuration; +import org.devlive.metaforge.service.itransient.configuration.ConnectionConfiguration; +import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; +import org.devlive.metaforge.service.itransient.configuration.NodeItemConfiguration; +import org.devlive.metaforge.service.itransient.configuration.PortConfiguration; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.stream.Collectors; + +/** + * The server is the authority for the node palette and executable graph shapes. Executors are + * discovered from the installed executor plugins; each one describes its capabilities, limits, job + * settings and node plugins in its own declaration, and compiles validated graphs itself. Nothing + * here depends on a particular executor. + */ +@Slf4j +@Component +public class WorkflowCatalog +{ + public static final int MAX_NODES = 100; + public static final int MAX_CONNECTIONS = 200; + public static final long MAX_TIMEOUT = 86400; + + private final PluginManager plugins; + // Declarations ship inside the plugin; a reloaded plugin brings a new service instance and is read again. + private final Map> declarations = Collections.synchronizedMap(new WeakHashMap<>()); + + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "The plugin manager is an intentionally shared Spring dependency") + public WorkflowCatalog(PluginManager plugins) + { + this.plugins = plugins; + } + + /** Resolves a saved data source code for a node field of type {@code source}. */ + @FunctionalInterface + public interface SourceResolver + { + ExecutorConfigure resolve(String code); + } + + /** An available executor: its plugin, service and declaration. */ + public static final class WorkflowExecutor + { + private final Plugin plugin; + private final ExecutorService service; + private final WorkflowExecutorDefinition definition; + + private WorkflowExecutor(Plugin plugin, ExecutorService service, WorkflowExecutorDefinition definition) + { + this.plugin = plugin; + this.service = service; + this.definition = definition; + } + + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "Plugins are shared runtime objects") + public Plugin plugin() + { + return plugin; + } + + public ExecutorService service() + { + return service; + } + + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "Declarations are read-only catalog data") + public WorkflowExecutorDefinition definition() + { + return definition; + } + } + + public Map catalog() + { + return Map.of("executors", executors().stream().map(WorkflowExecutor::definition).collect(Collectors.toList())); + } + + /** Executors whose plugin is running and ships a workflow declaration, ordered by name. */ + public List executors() + { + List result = new ArrayList<>(); + for (PluginMetadata metadata : plugins.getPluginInfos()) { + if (metadata.getType() == PluginType.EXECUTOR && metadata.getName() != null) { + find(metadata.getName()).ifPresent(result::add); + } + } + result.sort(Comparator.comparing(executor -> executor.definition().getName())); + return result; + } + + public WorkflowExecutor executor(String name) + { + return find(name).orElseThrow(() -> new IllegalArgumentException("Executor is unavailable: " + name)); + } + + public List definitions(String executor) + { + return executor(executor).definition().getNodes(); + } + + private Optional find(String name) + { + if (name == null || name.isBlank()) { + return Optional.empty(); + } + Optional plugin = plugins.getPlugin(name); + if (plugin.isEmpty() || plugin.get().getType() != PluginType.EXECUTOR) { + return Optional.empty(); + } + ExecutorService service; + try { + service = plugin.get().getService(ExecutorService.class); + } + catch (RuntimeException e) { + log.warn("Executor plugin {} has no executor service", name, e); + return Optional.empty(); + } + if (service == null) { + return Optional.empty(); + } + String executorName = plugin.get().getName(); + return declaration(executorName, service).map(definition -> new WorkflowExecutor(plugin.get(), service, definition)); + } + + private Optional declaration(String name, ExecutorService service) + { + return declarations.computeIfAbsent(service, ignored -> { + URL location = service.workflowDeclaration(); + if (location == null) { + return Optional.empty(); + } + try { + return Optional.of(WorkflowDeclarationReader.read(name, location)); + } + catch (IOException | RuntimeException e) { + log.warn("Cannot read the workflow declaration of executor {} at {}", name, location, e); + return Optional.empty(); + } + }); + } + + public List validate(WorkflowEntity workflow) + { + List errors = new ArrayList<>(); + try { + WorkflowExecutor executor = executor(workflow.getExecutor()); + WorkflowExecutorDefinition definition = executor.definition(); + Configuration graph = workflow.getConfigure(); + if (graph == null || graph.getNodes() == null || graph.getNodes().isEmpty()) { + return List.of("Add an input and an output node"); + } + if (graph.getNodes().size() > MAX_NODES || graph.getConnections() != null && graph.getConnections().size() > MAX_CONNECTIONS) { + return List.of("Workflow is too large"); + } + if (graph.getTimeout() != null && (graph.getTimeout() < 1 || graph.getTimeout() > MAX_TIMEOUT)) { + errors.add("Timeout must be between 1 and " + MAX_TIMEOUT + " seconds"); + } + errors.addAll(fieldErrors("Settings", definition.getSettings(), graph.getSettings())); + Map allowed = definition.getNodes().stream() + .collect(Collectors.toMap(NodeConfiguration::getId, node -> node, (left, right) -> left)); + Map nodes = new LinkedHashMap<>(); + for (NodeConfiguration node : graph.getNodes()) { + if (node.getId() == null || nodes.putIfAbsent(node.getId(), node) != null) { + errors.add("Node identifiers must be present and unique"); + } + NodeConfiguration type = allowed.get(node.getTid()); + if (type == null || !Objects.equals(type.getCategory(), node.getCategory()) || !Objects.equals(type.getKey(), node.getKey())) { + errors.add("Unsupported node for " + definition.getName() + ": " + node.getTid()); + continue; + } + errors.addAll(fieldErrors(label(node), type.getConfigure(), node.getData())); + } + for (String category : WorkflowExecutorDefinition.CATEGORIES) { + long count = graph.getNodes().stream().filter(node -> category.equals(node.getCategory())).count(); + int minimum = "transform".equals(category) ? 0 : 1; + if (count < minimum || count > definition.limit(category)) { + errors.add("source".equals(category) || "sink".equals(category) + ? "This executor requires " + (definition.limit(category) == 1 ? "exactly one " : "at least one ") + ("source".equals(category) ? "input" : "output") + : "This executor supports at most " + definition.limit(category) + " transforms"); + } + } + if (!errors.isEmpty()) { + return errors; + } + List chain = pipeline(graph, allowed, errors); + if (errors.isEmpty() && chain != null) { + errors.addAll(executor.service().validate(plan(workflow, chain, definition, null, "", "", Map.of()))); + } + } + catch (IllegalArgumentException e) { + errors.add(e.getMessage()); + } + return errors; + } + + /** + * Builds the plan an executor compiles. The workflow must have passed {@link #validate}; saved + * data sources referenced by {@code source} fields are resolved through {@code resolver}. + */ + public WorkflowPlan plan(WorkflowEntity workflow, String taskName, String userName, Map options, SourceResolver resolver) + { + WorkflowExecutorDefinition definition = executor(workflow.getExecutor()).definition(); + Map allowed = definition.getNodes().stream() + .collect(Collectors.toMap(NodeConfiguration::getId, node -> node, (left, right) -> left)); + List errors = new ArrayList<>(); + List chain = pipeline(workflow.getConfigure(), allowed, errors); + if (chain == null || !errors.isEmpty()) { + throw new IllegalArgumentException(String.join("\n", errors)); + } + return plan(workflow, chain, definition, resolver, taskName, userName, options); + } + + private WorkflowPlan plan(WorkflowEntity workflow, List chain, WorkflowExecutorDefinition definition, + SourceResolver resolver, String taskName, String userName, Map options) + { + Map types = definition.getNodes().stream() + .collect(Collectors.toMap(NodeConfiguration::getId, node -> node, (left, right) -> left)); + List nodes = new ArrayList<>(); + for (NodeConfiguration node : chain) { + Map values = node.getData() == null ? Map.of() : new LinkedHashMap<>(node.getData()); + Map sources = new LinkedHashMap<>(); + NodeConfiguration type = types.get(node.getTid()); + if (resolver != null && type != null) { + for (NodeItemConfiguration field : type.getConfigure()) { + Object code = values.get(field.getField()); + if ("source".equals(field.getType()) && code != null && !code.toString().isBlank()) { + sources.put(field.getField(), resolver.resolve(code.toString())); + } + } + } + WorkflowCategory category = WorkflowCategory.of(node.getCategory()); + if (category == null) { + throw new IllegalArgumentException("Unsupported node category: " + node.getCategory()); + } + nodes.add(new WorkflowNode(node.getId(), node.getTid(), node.getKey(), category, label(node), values, sources)); + } + Configuration graph = workflow.getConfigure(); + Map settings = graph.getSettings() == null ? Map.of() : new LinkedHashMap<>(graph.getSettings()); + long timeout = Optional.ofNullable(graph.getTimeout()).orElse(600L); + return new WorkflowPlan(taskName, userName, workflow.getWork(), timeout, nodes, settings, options); + } + + /** Orders a single input-to-output chain, reporting connection and topology errors. */ + private List pipeline(Configuration graph, Map types, List errors) + { + Map nodes = new LinkedHashMap<>(); + graph.getNodes().forEach(node -> nodes.putIfAbsent(node.getId(), node)); + Map incoming = new HashMap<>(); + Map outgoing = new HashMap<>(); + Map next = new HashMap<>(); + Set ports = new HashSet<>(); + for (ConnectionConfiguration connection : graph.getConnections() == null ? List.of() : graph.getConnections()) { + String source = endpoint(connection.getSource(), nodes, types, "output"); + String target = endpoint(connection.getTarget(), nodes, types, "input"); + if (source == null || target == null || source.equals(target)) { + errors.add("Connection must join an output port to a different input port"); + continue; + } + if (!ports.add(connection.getSource()) || !ports.add(connection.getTarget())) { + errors.add("Each port accepts a single connection"); + continue; + } + outgoing.merge(source, 1, Integer::sum); + incoming.merge(target, 1, Integer::sum); + next.put(source, target); + } + for (NodeConfiguration node : nodes.values()) { + int inputs = incoming.getOrDefault(node.getId(), 0); + int outputs = outgoing.getOrDefault(node.getId(), 0); + if (inputs != ("source".equals(node.getCategory()) ? 0 : 1) || outputs != ("sink".equals(node.getCategory()) ? 0 : 1)) { + errors.add(label(node) + ": connect all nodes in a single input-to-output pipeline"); + } + } + List chain = new ArrayList<>(); + Set visited = new HashSet<>(); + String cursor = nodes.values().stream().filter(node -> "source".equals(node.getCategory())).map(NodeConfiguration::getId).findFirst().orElse(null); + while (cursor != null && visited.add(cursor)) { + chain.add(nodes.get(cursor)); + cursor = next.get(cursor); + } + if (cursor != null || visited.size() != nodes.size()) { + errors.add("The workflow contains a cycle or disconnected nodes"); + } + return errors.isEmpty() ? chain : null; + } + + /** The node owning a port endpoint ({@code -}) of the given direction. */ + private String endpoint(String endpoint, Map nodes, Map types, String direction) + { + if (endpoint == null) { + return null; + } + for (NodeConfiguration node : nodes.values()) { + NodeConfiguration type = types.get(node.getTid()); + List declared = type == null || type.getPorts() == null ? List.of() : type.getPorts(); + for (PortConfiguration port : declared) { + if (direction.equals(port.getType()) && endpoint.equals(node.getId() + "-" + port.getId())) { + return node.getId(); + } + } + } + return null; + } + + /** Checks values against declared fields: required, options, types and numeric bounds. */ + static List fieldErrors(String owner, List fields, Map data) + { + List errors = new ArrayList<>(); + Map values = data == null ? Map.of() : data; + for (NodeItemConfiguration field : fields == null ? List.of() : fields) { + Object value = values.get(field.getField()); + String name = owner + ": " + (field.getLabel() == null ? field.getField() : field.getLabel()); + boolean empty = value == null || value.toString().isBlank() + || value instanceof List && ((List) value).isEmpty() + || value instanceof Map && ((Map) value).isEmpty(); + if (empty) { + if (field.isRequired()) { + errors.add(name + " is required"); + } + continue; + } + if (field.getOptions() != null && !field.getOptions().isEmpty() + && field.getOptions().stream().noneMatch(option -> Objects.equals(option.getValue(), value))) { + errors.add(name + " has an invalid value"); + } + switch (field.getType()) { + case "number": + if (!(value instanceof Number)) { + errors.add(name + " must be numeric"); + } + else if (field.getMin() != null && ((Number) value).doubleValue() < field.getMin() + || field.getMax() != null && ((Number) value).doubleValue() > field.getMax()) { + errors.add(name + " is out of range"); + } + break; + case "switch": + if (!(value instanceof Boolean)) { + errors.add(name + " must be boolean"); + } + break; + case "array": + if (!(value instanceof List)) { + errors.add(name + " must be a list"); + } + break; + case "map": + case "mapping": + if (!(value instanceof Map)) { + errors.add(name + " must be a map"); + } + break; + default: + break; + } + } + return errors; + } + + private static String label(NodeConfiguration node) + { + return node.getLabel() == null ? node.getTid() : node.getLabel(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowDeclarationReader.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowDeclarationReader.java new file mode 100644 index 0000000000..87f19c8634 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowDeclarationReader.java @@ -0,0 +1,99 @@ +package org.devlive.metaforge.service.workflow; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; +import org.devlive.metaforge.service.itransient.configuration.NodeItemConfiguration; +import org.devlive.metaforge.service.itransient.configuration.PortConfiguration; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Reads an executor's {@code workflow.yaml}. Node entries are paths relative to the declaration + * (or inline node objects), so each node plugin keeps its own file next to the declaration. + */ +public final class WorkflowDeclarationReader +{ + private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); + + private WorkflowDeclarationReader() + {} + + public static WorkflowExecutorDefinition read(String executor, URL declaration) + throws IOException + { + JsonNode root = tree(declaration); + if (!(root instanceof ObjectNode)) { + throw new IOException(declaration + " is not a workflow declaration"); + } + List nodes = new ArrayList<>(); + for (JsonNode entry : root.path("nodes")) { + NodeConfiguration node = entry.isTextual() + ? YAML.treeToValue(tree(new URL(declaration, entry.asText())), NodeConfiguration.class) + : YAML.treeToValue(entry, NodeConfiguration.class); + nodes.add(check(declaration, node)); + } + ((ObjectNode) root).remove("nodes"); + WorkflowExecutorDefinition definition = YAML.treeToValue(root, WorkflowExecutorDefinition.class); + definition.setName(executor); + definition.setNodes(nodes); + Set ids = new HashSet<>(); + for (NodeConfiguration node : nodes) { + if (!ids.add(node.getId())) { + throw new IOException(declaration + " declares the node " + node.getId() + " twice"); + } + } + return definition; + } + + private static NodeConfiguration check(URL declaration, NodeConfiguration node) + throws IOException + { + if (node.getId() == null || node.getKey() == null || !WorkflowExecutorDefinition.CATEGORIES.contains(node.getCategory())) { + throw new IOException(declaration + " declares a node without id, key or a known category: " + node.getId()); + } + if (node.getTid() == null) { + node.setTid(node.getId()); + } + if (node.getConfigure() == null) { + node.setConfigure(new ArrayList<>()); + } + if (node.getPorts() == null || node.getPorts().isEmpty()) { + // Ports follow from the category: inputs read, outputs write, transforms do both. + List ports = new ArrayList<>(); + if (!"source".equals(node.getCategory())) { + ports.add(new PortConfiguration("input", "input", "Input", true, null)); + } + if (!"sink".equals(node.getCategory())) { + ports.add(new PortConfiguration("output", "output", "Output", true, null)); + } + node.setPorts(ports); + } + for (NodeItemConfiguration field : node.getConfigure()) { + if (field.getField() == null || field.getType() == null) { + throw new IOException(declaration + " node " + node.getId() + " has a field without name or type"); + } + } + return node; + } + + private static JsonNode tree(URL url) + throws IOException + { + URLConnection connection = url.openConnection(); + // Plugins are reloaded from their jars; a cached jar would stay open and stale. + connection.setUseCaches(false); + try (InputStream stream = connection.getInputStream()) { + return YAML.readTree(stream); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowExecutorDefinition.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowExecutorDefinition.java new file mode 100644 index 0000000000..37a57827bf --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowExecutorDefinition.java @@ -0,0 +1,81 @@ +package org.devlive.metaforge.service.workflow; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import org.devlive.metaforge.service.itransient.configuration.NodeConfiguration; +import org.devlive.metaforge.service.itransient.configuration.NodeItemConfiguration; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * What an executor plugin declares about workflows in its {@code workflow.yaml}: how runs can be + * observed and stopped, which graph shapes it executes, the job settings a workflow may set and the + * node plugins it offers. The editor and the server both follow this declaration. + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class WorkflowExecutorDefinition +{ + public static final String PIPELINE = "pipeline"; + public static final List CATEGORIES = List.of("source", "transform", "sink"); + + private String name; + private String description; + /** Graph shape the executor runs; only a single input-to-output {@value #PIPELINE} for now. */ + private String topology = PIPELINE; + private Capabilities capabilities = new Capabilities(); + /** Maximum number of nodes per category, keyed by {@code sources}, {@code transforms} and {@code sinks}. */ + private Map limits = new LinkedHashMap<>(); + private List settings = new ArrayList<>(); + /** + * Metrics the executor actually reports for a run, named after {@code ExecutorMetrics} fields + * (plus {@code stages} and {@code series}). Pages show exactly these, so a change to what an + * executor reports must update this list and the editor's metric registry together. + */ + private List metrics = new ArrayList<>(); + /** Relative node file paths in the declaration; resolved node definitions once read. */ + private List nodes = new ArrayList<>(); + + public List getCategories() + { + return CATEGORIES; + } + + public int limit(String category) + { + Integer value = limits.get(category + "s"); + return value == null ? ("transform".equals(category) ? 0 : 1) : Math.max(0, value); + } + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Capabilities + { + public static final String STOP_COOPERATIVE = "cooperative"; + public static final String STOP_INTERRUPT = "interrupt"; + public static final String STOP_NONE = "none"; + + /** {@value #STOP_COOPERATIVE}, {@value #STOP_INTERRUPT} or {@value #STOP_NONE}. */ + private String stop = STOP_INTERRUPT; + /** The executor reports how many rows it wrote; otherwise the count stays unknown. */ + private boolean rowCount; + private boolean metrics; + /** The executor returns a bounded sample of written rows. */ + private boolean preview; + + public boolean cooperativeStop() + { + return STOP_COOPERATIVE.equals(stop); + } + + public boolean stoppable() + { + return !STOP_NONE.equals(stop); + } + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowRun.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowRun.java new file mode 100644 index 0000000000..9563581d45 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/workflow/WorkflowRun.java @@ -0,0 +1,48 @@ +package org.devlive.metaforge.service.workflow; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import org.devlive.metaforge.executor.common.RunState; +import org.devlive.metaforge.executor.configure.ExecutorMetrics; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class WorkflowRun +{ + private String id; + private String code; + private String name; + private String executor; + private volatile RunState state; + private long startedAt; + private Long finishedAt; + private long elapsed; + private String message; + private Integer count; + private volatile ExecutorMetrics metrics; + private List> nodes; + private List> preview; + + /** Start time as the server formats every date (yyyy-MM-dd HH:mm:ss); the epoch value stays for computation. */ + @JsonProperty(access = JsonProperty.Access.READ_ONLY) + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss") + public Date getStartTime() + { + return startedAt > 0 ? new Date(startedAt) : null; + } + + @JsonProperty(access = JsonProperty.Access.READ_ONLY) + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss") + public Date getFinishTime() + { + return finishedAt == null ? null : new Date(finishedAt); + } +} diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java index 14924f7f64..ff31cf1d7a 100644 --- a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java @@ -2,11 +2,16 @@ import org.devlive.metaforge.plugin.configure.PluginConfigure; import org.devlive.metaforge.plugin.configure.PluginConfigureExecutor; +import org.devlive.metaforge.plugin.configure.PluginConfigureField; +import org.devlive.metaforge.plugin.configure.PluginConfigureFieldName; +import org.devlive.metaforge.plugin.configure.PluginConfigureFieldType; +import org.devlive.metaforge.spi.model.RuntimeConfigure; import org.devlive.metaforge.plugin.configure.PluginConfigurePipelineType; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class ConfigureUtilsTest @@ -61,4 +66,28 @@ public void testFindPipelineExecutorWithNullArguments() Assert.assertNull(ConfigureUtils.findPipelineExecutor(configure, null, null)); Assert.assertSame(matched, ConfigureUtils.findPipelineExecutor(configure, "TRUNCATE", PluginConfigurePipelineType.INPUT)); } + + @Test + public void testFieldsSentWithoutValueAreNotApplied() + { + // Testing a connection whose custom configures were sent as null must not fail with an NPE. + List fields = Arrays.asList( + PluginConfigureField.builder().field(PluginConfigureFieldName.host).type(PluginConfigureFieldType.String).value("127.0.0.1").build(), + PluginConfigureField.builder().field(PluginConfigureFieldName.port).type(PluginConfigureFieldType.Number).build(), + PluginConfigureField.builder().field(PluginConfigureFieldName.configures).type(PluginConfigureFieldType.Array).build()); + + List applied = ConfigureUtils.filterNotEmpty(fields); + Assert.assertEquals(1, applied.size()); + RuntimeConfigure configure = ConfigureUtils.preparedConfigure(applied); + Assert.assertEquals("127.0.0.1", configure.getHost()); + } + + @Test + public void testNullArrayConfiguresGiveNoEnvironment() + { + List fields = Arrays.asList( + PluginConfigureField.builder().field(PluginConfigureFieldName.configures).type(PluginConfigureFieldType.Array).build()); + + Assert.assertTrue(ConfigureUtils.getMapValue(fields, PluginConfigureFieldName.configures).isEmpty()); + } } diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/notify/UserNotifySettingsTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/notify/UserNotifySettingsTest.java new file mode 100644 index 0000000000..8b20fa2112 --- /dev/null +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/notify/UserNotifySettingsTest.java @@ -0,0 +1,193 @@ +package org.devlive.metaforge.service.notify; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.notify.NotifyService; +import org.devlive.metaforge.notify.model.NotifyRequest; +import org.devlive.metaforge.notify.model.NotifyResponse; +import org.devlive.metaforge.service.entity.NotificationEntity; +import org.devlive.metaforge.service.entity.RoleEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.itransient.NotificationTypeEntity; +import org.devlive.metaforge.service.repository.NotificationRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class UserNotifySettingsTest +{ + private static final String ROBOT = "DingTalk"; + + private final List saved = new ArrayList<>(); + private final List sent = new ArrayList<>(); + private UserEntity stored; + private UserNotifySettings settings; + + /** One plugin channel with a required secret and an optional URL. */ + private final class FakeChannels + extends NotificationChannels + { + private FakeChannels() + { + super(null); + } + + @Override + public List list() + { + return List.of(new Channel(INTERNAL, true, List.of()), + new Channel(ROBOT, false, List.of( + new Field("accessToken", "Access token", "SECRET", true, null, null), + new Field("callback", "Callback", "URL", false, null, null)))); + } + + @Override + public Optional service(String type) + { + if (!ROBOT.equals(type)) { + return Optional.empty(); + } + return Optional.of(request -> { + sent.add(request); + NotifyResponse response = new NotifyResponse(); + response.setSuccessful(true); + return response; + }); + } + } + + @SuppressWarnings("unchecked") + private T fake(Class type) + { + return (T) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[] {type}, (proxy, method, args) -> { + switch (method.getName()) { + case "findByCode": + return Optional.ofNullable(stored); + case "save": + if (args[0] instanceof NotificationEntity) { + saved.add((NotificationEntity) args[0]); + } + else { + stored = (UserEntity) args[0]; + } + return args[0]; + case "hashCode": + return 0; + case "equals": + return proxy == args[0]; + case "toString": + return type.getSimpleName(); + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + @Before + public void setUp() + { + stored = new UserEntity(); + stored.setId(1L); + stored.setCode("u1"); + UserRepository users = fake(UserRepository.class); + NotificationChannels channels = new FakeChannels(); + settings = new UserNotifySettings(users, channels, new NotificationDispatcher(users, fake(NotificationRepository.class), channels)); + } + + private static UserEntity login(String... roles) + { + UserEntity user = new UserEntity(); + user.setCode("u1"); + user.setRoles(java.util.Arrays.stream(roles).map(code -> RoleEntity.builder().code(code).build()).collect(Collectors.toSet())); + return user; + } + + private static UserNotifySettings.ChannelSetting robot(boolean enabled, Map configure, Set services) + { + return new UserNotifySettings.ChannelSetting(ROBOT, enabled, services, configure, null); + } + + @Test + public void enabledChannelNeedsItsRequiredSettings() + { + CommonResponse response = settings.save(login(), List.of(robot(true, Map.of(), Set.of("DATASET_CREATED")))); + Assert.assertFalse(response.getStatus()); + Assert.assertTrue(response.getMessage().toString(), response.getMessage().toString().contains("required")); + } + + @Test + public void secretsAreWriteOnlyAndEmptyKeepsTheStoredValue() + { + Assert.assertTrue(settings.save(login(), List.of(robot(true, Map.of("accessToken", "tok-1"), Set.of("DATASET_CREATED")))).getStatus()); + Assert.assertEquals("tok-1", stored.getNotifyConfigure().get(0).getConfigure().get("accessToken")); + + UserNotifySettings.ChannelSetting read = settings.settings(login()).getData().stream() + .filter(channel -> ROBOT.equals(channel.getType())).findFirst().orElseThrow(); + Assert.assertEquals("", read.getConfigure().get("accessToken")); + Assert.assertEquals(Set.of("accessToken"), read.getConfigured()); + + Assert.assertTrue(settings.save(login(), List.of(robot(true, Map.of("accessToken", ""), Set.of("DATASET_UPDATED")))).getStatus()); + NotificationTypeEntity entry = stored.getNotifyConfigure().get(0); + Assert.assertEquals("tok-1", entry.getConfigure().get("accessToken")); + Assert.assertEquals(Set.of("DATASET_UPDATED"), entry.getServices()); + } + + @Test + public void disabledChannelKeepsSettingsButDropsSubscriptions() + { + settings.save(login(), List.of(robot(true, Map.of("accessToken", "tok-1"), Set.of("DATASET_CREATED")))); + Assert.assertTrue(settings.save(login(), List.of(robot(false, Map.of(), Set.of("DATASET_CREATED")))).getStatus()); + NotificationTypeEntity entry = stored.getNotifyConfigure().get(0); + Assert.assertFalse(entry.isEnabled()); + Assert.assertTrue(entry.getServices().isEmpty()); + Assert.assertEquals("tok-1", entry.getConfigure().get("accessToken")); + } + + @Test + public void unknownChannelsScenesAndBadAddressesAreRejected() + { + Assert.assertFalse(settings.save(login(), List.of(new UserNotifySettings.ChannelSetting("Nope", true, Set.of(), Map.of(), null))).getStatus()); + Assert.assertFalse(settings.save(login(), List.of(robot(true, Map.of("accessToken", "t"), Set.of("NOPE")))).getStatus()); + Assert.assertFalse(settings.save(login(), List.of(robot(true, Map.of("accessToken", "t", "callback", "ftp://x"), Set.of()))).getStatus()); + } + + @Test + public void adminScenesAreOfferedToAdministratorsOnly() + { + Assert.assertFalse(settings.save(login(), List.of(robot(true, Map.of("accessToken", "t"), Set.of("PLUGIN_INSTALLED")))).getStatus()); + Assert.assertTrue(settings.save(login("ADMIN"), List.of(robot(true, Map.of("accessToken", "t"), Set.of("PLUGIN_INSTALLED")))).getStatus()); + Assert.assertTrue(settings.catalog(login()).getCategories().stream().noneMatch(category -> "PLUGIN".equals(category.getKey()))); + Assert.assertTrue(settings.catalog(login("ADMIN")).getCategories().stream().anyMatch(category -> "PLUGIN".equals(category.getKey()))); + } + + @Test + public void legacySubscriptionsReadAsScenes() + { + stored.setNotifyConfigure(new ArrayList<>(List.of(NotificationTypeEntity.builder() + .type(NotificationChannels.INTERNAL).enabled(true).services(Set.of("DYNAMIC", "SYNCDATA")).build()))); + Set services = settings.settings(login()).getData().get(0).getServices(); + Assert.assertEquals(Set.of("DATASET_CREATED", "DATASET_UPDATED", "DATASET_SYNC_SUCCEEDED", "DATASET_SYNC_FAILED"), services); + Assert.assertEquals(List.of(NotificationChannels.INTERNAL), NotificationDispatcher.subscribed(stored, NotificationScene.DATASET_SYNC_FAILED)); + Assert.assertTrue(NotificationDispatcher.subscribed(stored, NotificationScene.SOURCE_CREATED).isEmpty()); + } + + @Test + public void testSendUsesTheStoredSecretWhenLeftEmpty() + { + settings.save(login(), List.of(robot(true, Map.of("accessToken", "tok-1"), Set.of()))); + Assert.assertTrue(settings.test(login(), robot(true, Map.of(), Set.of())).getStatus()); + Assert.assertEquals("tok-1", sent.get(0).getConfigure().get("accessToken")); + + Assert.assertTrue(settings.test(login(), new UserNotifySettings.ChannelSetting(NotificationChannels.INTERNAL, true, Set.of(), Map.of(), null)).getStatus()); + Assert.assertEquals(1, saved.size()); + } +} diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSqlTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSqlTest.java new file mode 100644 index 0000000000..e4def4233b --- /dev/null +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReadOnlyChartSqlTest.java @@ -0,0 +1,43 @@ +package org.devlive.metaforge.service.service.impl; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ReadOnlyChartSqlTest +{ + @Test + public void acceptsOneSelectWithAggregatesAndQuotedDelimiters() + { + for (String sql : List.of( + " SELECT COUNT(*) AS total FROM `events` WHERE message = 'it''s a ; -- value'; ", + "SELECT CAST(score AS DECIMAL(10, 2)) FROM \"scores\" WHERE score IN (1, 2)", + "SELECT DISTINCT account_id FROM events WHERE EXISTS (SELECT 1 FROM accounts)")) { + assertTrue(sql, ReadOnlyChartSql.accepts(sql)); + } + } + + @Test + public void rejectsScriptsWritesAndCallsWithSideEffects() + { + for (String sql : List.of( + "DELETE FROM events", + "SELECT * FROM events; DELETE FROM events", + "SELECT 'unclosed", + "SELECT 'escape\\' ; SELECT 2", + "SELECT * FROM events FOR UPDATE", + "SELECT * INTO OUTFILE '/tmp/events' FROM events", + "SELECT * INTO new_table FROM events", + "SELECT /*!50000 GET_LOCK('chart', 0) */ 1", + "SELECT get_lock('chart', 0)", + "SELECT pg_catalog.count(id) FROM events", + "SELECT \"count\"(id) FROM events", + "SELECT @variable := 1", + "SELECT $$unexpected$$")) { + assertFalse(sql, ReadOnlyChartSql.accepts(sql)); + } + } +} diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java new file mode 100644 index 0000000000..ca121104af --- /dev/null +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java @@ -0,0 +1,326 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.entity.DataSetEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.NotificationScene; +import org.devlive.metaforge.service.enums.ReportType; +import org.devlive.metaforge.service.event.NotificationEvent; +import org.devlive.metaforge.service.event.NotificationEventPublisher; +import org.devlive.metaforge.service.repository.DataSetRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.repository.jdbc.LinkRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DataSetService; +import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class ReportShareServiceImplTest +{ + private static final long OWNER = 7L; + private static final String CHART = "{\"queryChart\":{\"version\":1},\"columns\":[]}"; + + private final List reports = new ArrayList<>(); + private final AtomicInteger adhocCalls = new AtomicInteger(); + private final AtomicReference adhocUser = new AtomicReference<>(); + private ReportShareServiceImpl service; + + @SuppressWarnings("unchecked") + private static T fake(Class type, java.util.function.BiFunction handler) + { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return "equals".equals(method.getName()) ? proxy == args[0] : "hashCode".equals(method.getName()) ? System.identityHashCode(proxy) : type.getSimpleName(); + } + if (method.isDefault()) { + return java.lang.invoke.MethodHandles.privateLookupIn(type, java.lang.invoke.MethodHandles.lookup()) + .unreflectSpecial(method, type).bindTo(proxy).invokeWithArguments(args == null ? new Object[0] : args); + } + return handler.apply(method.getName(), args); + }); + } + + private ReportRepository reportRepository() + { + return fake(ReportRepository.class, (name, args) -> { + switch (name) { + case "findByCode": + return reports.stream().filter(r -> Objects.equals(r.getCode(), args[0])).findFirst(); + case "findByShareToken": + return reports.stream().filter(r -> Objects.equals(r.getShareToken(), args[0])).findFirst(); + case "findById": + return reports.stream().filter(r -> Objects.equals(r.getId(), args[0])).findFirst(); + case "save": + return args[0]; + default: + throw new UnsupportedOperationException(name); + } + }); + } + + @Before + public void initialize() + { + UserEntity owner = new UserEntity(); + owner.setId(OWNER); + owner.setUsername("owner"); + UserRepository users = fake(UserRepository.class, (name, args) -> Objects.equals(args[0], OWNER) ? Optional.of(owner) : Optional.empty()); + DataSetEntity dataset = new DataSetEntity(); + dataset.setId(3L); + dataset.setCode("ds"); + DataSetRepository datasets = fake(DataSetRepository.class, (name, args) -> Optional.of(dataset)); + DataSetService dataSetService = fake(DataSetService.class, (name, args) -> { + adhocCalls.incrementAndGet(); + adhocUser.set(UserDetailsService.getUser().getId()); + return CommonResponse.success(Map.of("columns", List.of())); + }); + ReportService reportService = fake(ReportService.class, (name, args) -> CommonResponse.success(List.of())); + service = new ReportShareServiceImpl(reportRepository(), users, datasets, reportService, dataSetService, new ObjectMapper()); + signIn(OWNER); + } + + @After + public void close() + { + SecurityContextHolder.clearContext(); + } + + private static void signIn(Long id) + { + UserDetailsService principal = new UserDetailsService(id, "u" + id, "u" + id, "", new ArrayList<>(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); + } + + private ReportEntity report(String code, ReportType type, String configure) + { + ReportEntity report = new ReportEntity(); + report.setId((long) reports.size() + 1); + report.setCode(code); + report.setName(code); + report.setUserId(OWNER); + report.setType(type); + report.setConfigure(configure); + report.setDatasetId(3L); + report.setQuery("{\"columns\":[],\"limit\":10}"); + reports.add(report); + return report; + } + + private static ReportShareService.ShareRequest request(boolean enabled, boolean embed, String origins, boolean rotate) + { + return new ReportShareService.ShareRequest(enabled, embed, origins, rotate); + } + + @Test + public void onlyTheOwnerSharesAndTheLinkCanBeRotated() + { + report("chart", ReportType.QUERY, CHART); + signIn(99L); + assertFalse(service.share("chart", request(true, false, "", false)).getStatus()); + signIn(OWNER); + CommonResponse shared = service.share("chart", request(true, false, "", false)); + assertTrue(shared.getStatus()); + String token = shared.getData().getToken(); + assertTrue(ReportShareServiceImpl.TOKEN.matcher(token).matches()); + assertEquals(token, service.share("chart", request(true, false, "", false)).getData().getToken()); + String rotated = service.share("chart", request(true, false, "", true)).getData().getToken(); + assertNotEquals(token, rotated); + assertFalse("the old link stops working", service.view(token).getStatus()); + assertTrue(service.view(rotated).getStatus()); + } + + @Test + public void turningSharingOnOrOffNotifiesTheOwnerOnce() + { + List scenes = new ArrayList<>(); + service.setNotifications(new NotificationEventPublisher(event -> scenes.add(((NotificationEvent) event).getScene()))); + report("chart", ReportType.QUERY, CHART); + service.share("chart", request(true, false, "", false)); + service.share("chart", request(true, false, "", true)); + service.share("chart", request(false, false, "", false)); + assertEquals("rotating the link is not a new share", List.of(NotificationScene.REPORT_SHARED, NotificationScene.REPORT_UNSHARED), scenes); + } + + @Test + public void earlierQueryReportsCannotBeShared() + { + report("legacy", ReportType.QUERY, "{\"type\":\"LINE\"}"); + assertFalse(service.share("legacy", request(true, false, "", false)).getStatus()); + } + + @Test + public void thePublicViewHidesSqlSourceAndOwner() + { + ReportEntity report = report("chart", ReportType.QUERY, CHART); + report.setDescription("Weekly"); + String token = service.share("chart", request(true, false, "", false)).getData().getToken(); + ReportShareService.PublicReport view = service.view(token).getData(); + assertEquals("chart", view.getName()); + assertEquals(CHART, view.getConfigure()); + assertEquals("QUERY", view.getType()); + assertFalse(view.isEmbed()); + service.share("chart", request(false, false, "", false)); + assertFalse("disabled links are gone", service.view(token).getStatus()); + assertFalse(service.view("short").getStatus()); + assertFalse(service.view(null).getStatus()); + } + + @Test + public void framingFollowsTheOwnersChoice() + { + report("chart", ReportType.QUERY, CHART); + String token = service.share("chart", request(true, false, "", false)).getData().getToken(); + assertFalse(service.frameAncestors(token).isPresent()); + service.share("chart", request(true, true, "", false)); + assertEquals("*", service.frameAncestors(token).orElse(null)); + service.share("chart", request(true, true, "https://a.example.com/\nhttps://a.example.com, https://*.b.org:8443", false)); + assertEquals("'self' https://a.example.com https://*.b.org:8443", service.frameAncestors(token).orElse(null)); + assertFalse(service.share("chart", request(true, true, "javascript:alert(1)", false)).getStatus()); + assertFalse(service.share("chart", request(true, true, "https://a.com/path", false)).getStatus()); + } + + @Test + public void datasetDataRunsAsTheOwnerAndIsCached() + { + report("ds-report", ReportType.DATASET, "{}"); + String token = service.share("ds-report", request(true, false, "", false)).getData().getToken(); + SecurityContextHolder.clearContext(); + assertTrue(service.data(token).getStatus()); + assertEquals(Long.valueOf(OWNER), adhocUser.get()); + assertNull("the visitor stays anonymous afterwards", SecurityContextHolder.getContext().getAuthentication()); + assertTrue(service.data(token).getStatus()); + assertEquals("public views share one cached result", 1, adhocCalls.get()); + assertFalse("a dataset report has no chart refresh", service.refresh(token).getStatus()); + } + + @Test + public void savingAReportNeverChangesItsSharing() + { + SourceRepository sources = fake(SourceRepository.class, (name, args) -> Optional.empty()); + DataSetEntity dataset = new DataSetEntity(); + dataset.setCode("ds"); + DataSetRepository datasets = fake(DataSetRepository.class, (name, args) -> Optional.of(dataset)); + LinkRepository links = null; + ReportServiceImpl reportService = new ReportServiceImpl(reportRepository(), sources, datasets, links, new QueryChartRefresh(new ObjectMapper(), null)); + + ReportEntity created = new ReportEntity(); + created.setDataset(dataset); + created.setShareEnabled(true); + created.setShareToken("chosen-by-the-client-000000000000"); + ReportEntity saved = reportService.saveOrUpdate(fake(ReportRepository.class, (name, args) -> args[0]), created).getData(); + assertFalse(saved.isShareEnabled()); + assertNull(saved.getShareToken()); + + ReportEntity stored = report("ds-report", ReportType.DATASET, "{}"); + String token = service.share("ds-report", request(true, true, "https://a.com", false)).getData().getToken(); + ReportEntity update = new ReportEntity(); + update.setId(stored.getId()); + update.setDataset(dataset); + ReportEntity kept = reportService.saveOrUpdate(fake(ReportRepository.class, (name, args) -> args[0]), update).getData(); + assertNotNull(kept); + assertTrue(kept.isShareEnabled()); + assertTrue(kept.isShareEmbed()); + assertEquals(token, kept.getShareToken()); + assertEquals("https://a.com", kept.getShareOrigins()); + assertTrue(kept.isShareShowTitle()); + } + + @Test + public void internalErrorsNeverReachAnonymousVisitors() + { + ReportRepository broken = fake(ReportRepository.class, (name, args) -> { + throw new IllegalStateException("bad SQL grammar [SELECT secret FROM metaforge_report]"); + }); + ReportShareServiceImpl failing = new ReportShareServiceImpl(broken, null, null, null, null, new ObjectMapper()); + String token = "abcdefghijklmnopqrstuvwxyz012-_A"; + CommonResponse view = failing.view(token); + assertFalse(view.getStatus()); + assertFalse(String.valueOf(view.getMessage()).contains("SQL")); + assertFalse(failing.frameAncestors(token).isPresent()); + assertFalse(failing.data(token).getStatus()); + } + + @Test + public void linksExpireAndTheOwnerChoosesWhatThePageShows() + { + ReportEntity report = report("chart", ReportType.QUERY, CHART); + report.setDescription("Weekly"); + long past = System.currentTimeMillis() - 1000; + assertFalse("an expiry in the past is refused", + service.share("chart", new ReportShareService.ShareRequest(true, true, "", false, past, null, null)).getStatus()); + long future = System.currentTimeMillis() + 3_600_000; + CommonResponse shared = service.share("chart", new ReportShareService.ShareRequest(true, true, "", false, future, false, false)); + assertTrue(shared.getStatus()); + assertEquals(Long.valueOf(future), shared.getData().getExpiresAt()); + String token = shared.getData().getToken(); + ReportShareService.PublicReport view = service.view(token).getData(); + assertFalse(view.isShowTitle()); + assertFalse(view.isShowDescription()); + assertNull("a hidden description is not sent", view.getDescription()); + + report.setShareExpiresAt(new java.util.Date(past)); + assertFalse("expired links stop working", service.view(token).getStatus()); + assertFalse(service.frameAncestors(token).isPresent()); + assertTrue("an owner can extend the link", service.share("chart", new ReportShareService.ShareRequest(true, true, "", false, null, null, null)).getStatus()); + ReportShareService.PublicReport again = service.view(token).getData(); + assertTrue("omitted display options keep the page complete", again.isShowTitle() && again.isShowDescription()); + assertEquals("Weekly", again.getDescription()); + } + + @Test + public void dataTablesAndDownloadsNeedTheOwnersPermission() + { + report("chart", ReportType.QUERY, "{\"queryChart\":{\"version\":1},\"columns\":[{\"region\":\"East\",\"amount\":12},{\"region\":\"West\",\"amount\":7}]}"); + String token = service.share("chart", request(true, false, "", false)).getData().getToken(); + assertFalse("off by default", service.table(token).getStatus()); + assertFalse(service.download(token).isPresent()); + + service.share("chart", new ReportShareService.ShareRequest(true, false, "", false, null, null, null, true, false)); + ReportShareService.Table table = service.table(token).getData(); + assertEquals(List.of("region", "amount"), table.getHeaders()); + assertEquals(List.of(List.of("East", 12), List.of("West", 7)), table.getRows()); + assertFalse("viewing data does not allow downloading it", service.download(token).isPresent()); + + service.share("chart", new ReportShareService.ShareRequest(true, false, "", false, null, null, null, false, true)); + assertFalse(service.table(token).getStatus()); + Map.Entry download = service.download(token).orElseThrow(); + assertEquals("chart", download.getKey()); + assertEquals(2, download.getValue().getRows().size()); + assertTrue(service.view(token).getData().isAllowDownload()); + } + + @Test + public void datasetRowsKeepHeaderOrderWhetherListsOrObjects() + { + ObjectMapper mapper = new ObjectMapper(); + ReportShareService.Table table = ReportShareServiceImpl.table(List.of("a", "b"), List.of( + mapper.valueToTree(Map.of("b", 2, "a", "x")), mapper.valueToTree(List.of("y", 3)))); + assertEquals(List.of(List.of("x", 2), List.of("y", 3)), table.getRows()); + assertFalse(table.isTruncated()); + } +} diff --git a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/PluginService.java b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/PluginService.java index 473df138b3..35cdc9a477 100644 --- a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/PluginService.java +++ b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/PluginService.java @@ -125,16 +125,6 @@ default String connectType() return "metaforge"; } - /** - * 是否支持元数据管理 - * - * @return 是否支持 - */ - default Boolean isSupportMeta() - { - return false; - } - default void connect(RuntimeConfigure configure) { Response response = new Response(); @@ -693,6 +683,14 @@ default Response metadataColumns(RuntimeConfigure configure, String database, St catch (SQLException ignored) { decimalDigits = 0; } + // JDBC 4.1 column; drivers such as MonetDB's do not return it + String autoincrement; + try { + autoincrement = resultSet.getString("IS_AUTOINCREMENT"); + } + catch (SQLException ignored) { + autoincrement = "NO"; + } rows.add(Lists.newArrayList( "column", resultSet.getString("COLUMN_NAME"), @@ -703,24 +701,29 @@ default Response metadataColumns(RuntimeConfigure configure, String database, St resultSet.getInt("ORDINAL_POSITION"), "", columnSize, - resultSet.getString("IS_AUTOINCREMENT"), + autoincrement, decimalDigits)); } } + // Primary keys are extra detail; servers such as Kyuubi's Spark engine do not support the lookup + List primaryColumns = Lists.newArrayList(); try (ResultSet resultSet = metaData.getPrimaryKeys(null, database, table)) { - List primaryColumns = Lists.newArrayList(); while (resultSet.next()) { primaryColumns.add(resultSet.getString("COLUMN_NAME")); } - for (String primaryColumn : primaryColumns) { - rows.add(Lists.newArrayList( - "primary", - primaryColumn, - "", "", "", "", - 0, - "PRIMARY KEY on (" + String.join(", ", primaryColumns) + ")", - 0, "NO", 0)); - } + } + catch (SQLException ex) { + log.warn("Primary keys of {}.{} are not available: {}", database, table, ex.getMessage()); + primaryColumns.clear(); + } + for (String primaryColumn : primaryColumns) { + rows.add(Lists.newArrayList( + "primary", + primaryColumn, + "", "", "", "", + 0, + "PRIMARY KEY on (" + String.join(", ", primaryColumns) + ")", + 0, "NO", 0)); } response.setColumns(formatRows(configure, response, rows)); response.setIsConnected(true); diff --git a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/BatchWriter.java b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/BatchWriter.java index e2efdfff32..1c41fa9f5b 100644 --- a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/BatchWriter.java +++ b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/BatchWriter.java @@ -7,8 +7,23 @@ public interface BatchWriter { void addRow(List row); + /** + * 立即提交已攒下的行。没有待提交的行时什么也不做。 + * Commit the rows added so far; does nothing when none are pending. + */ + void flush(); + long writtenCount(); + /** + * 写入失败的一批是否不留下任何行,因而可以安全地重写其中一部分。 + * Whether a failed write leaves none of its rows behind, so part of it can safely be written again. + */ + default boolean failedWritesLeaveNoRows() + { + return false; + } + @Override void close(); } diff --git a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/JdbcStreamAdapter.java b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/JdbcStreamAdapter.java index cf30a3eac6..07388da309 100644 --- a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/JdbcStreamAdapter.java +++ b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/adapter/JdbcStreamAdapter.java @@ -12,6 +12,8 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.SQLRecoverableException; import java.sql.Statement; import java.util.ArrayList; import java.util.List; @@ -118,7 +120,13 @@ public static BatchWriter openBatchWriter( throw new IllegalStateException("Open jdbc connection failed: " + Optional.ofNullable(jdbcConnection.getResponse().getMessage()).orElse("unknown")); } - return new JdbcBatchWriter(jdbcConnection, connection, database, table, columns, effectiveBatchSize); + return batchWriter(connection, jdbcConnection::destroy, database, table, columns, effectiveBatchSize); + } + + /** A batch writer on an open connection; {@code release} runs once the writer is closed. */ + static BatchWriter batchWriter(Connection connection, Runnable release, String database, String table, List columns, int batchSize) + { + return new JdbcBatchWriter(release, connection, database, table, columns, batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE); } private static JdbcConnection openConnection(PluginService plugin, RuntimeConfigure configure) @@ -157,7 +165,7 @@ private static void applyFetchSize(Statement statement, int fetchSize, RuntimeCo private static final class JdbcBatchWriter implements BatchWriter { - private final JdbcConnection jdbcConnection; + private final Runnable release; private final Connection connection; private final PreparedStatement statement; private final int columnCount; @@ -166,31 +174,43 @@ private static final class JdbcBatchWriter private long written; private boolean originalAutoCommit = true; private boolean restoreAutoCommit; + /** + * Whether each batch is committed in a transaction. Engines without transactions (ClickHouse + * and other analytical stores) write on executeBatch and reject commit(), so they run with + * the driver's auto-commit instead. + */ + private boolean transactional; JdbcBatchWriter( - JdbcConnection jdbcConnection, + Runnable release, Connection connection, String database, String table, List columns, int batchSize) { - this.jdbcConnection = jdbcConnection; + this.release = release; this.connection = connection; this.columnCount = columns.size(); this.batchSize = batchSize; try { this.originalAutoCommit = connection.getAutoCommit(); - if (this.originalAutoCommit) { - connection.setAutoCommit(false); - this.restoreAutoCommit = true; + if (supportsTransactions(connection)) { + if (this.originalAutoCommit) { + connection.setAutoCommit(false); + this.restoreAutoCommit = true; + } + this.transactional = true; } } - catch (SQLException ignore) { + catch (SQLException ex) { + // A driver that cannot switch auto-commit off keeps writing with auto-commit. + log.debug("Batch writer falls back to auto-commit: {}", ex.getMessage()); + this.transactional = false; } - String sql = buildInsertTemplate(database, table, columns); + String sql = buildInsertTemplate(identifierQuote(connection), database, table, columns); try { this.statement = connection.prepareStatement(sql); } @@ -220,14 +240,51 @@ public void addRow(List row) statement.addBatch(); pending++; if (pending >= batchSize) { - flush(); + commitPending(); } } catch (SQLException ex) { + // Rows bound so far must not leak into the caller's next attempt. + discardFailedBatch(); throw new IllegalStateException("Add batch row failed: " + ex.getMessage(), ex); } } + @Override + public void flush() + { + if (pending == 0) { + return; + } + try { + commitPending(); + } + catch (SQLException ex) { + throw new IllegalStateException("Flush batch failed: " + ex.getMessage(), ex); + } + } + + /** + * Transactions roll back; a ClickHouse INSERT is parsed before it is written, so a rejected + * one writes nothing. Other engines without transactions may keep part of a failed batch. + */ + @Override + public boolean failedWritesLeaveNoRows() + { + return transactional || clickHouse(); + } + + private boolean clickHouse() + { + try { + String product = connection.getMetaData().getDatabaseProductName(); + return product != null && product.toLowerCase(java.util.Locale.ROOT).contains("clickhouse"); + } + catch (SQLException | RuntimeException ex) { + return false; + } + } + @Override public long writtenCount() { @@ -239,7 +296,7 @@ public void close() { try { if (pending > 0) { - flush(); + commitPending(); } } catch (SQLException ex) { @@ -250,16 +307,66 @@ public void close() } } - private void flush() + private void commitPending() throws SQLException { - statement.executeBatch(); - connection.commit(); + try { + statement.executeBatch(); + if (transactional) { + try { + connection.commit(); + } + catch (SQLFeatureNotSupportedException ex) { + // The driver advertised transactions but cannot commit: the batch is already written. + log.warn("Driver does not support commit(); continuing with auto-commit: {}", ex.getMessage()); + transactional = false; + try { + connection.setAutoCommit(true); + restoreAutoCommit = !originalAutoCommit; + } + catch (SQLException ignored) { + // Nothing more to do: the driver writes on executeBatch. + } + } + } + } + catch (SQLException ex) { + discardFailedBatch(); + throw ex; + } + catch (AssertionError ex) { + // pgjdbc reports a connection lost during executeBatch with an AssertionError. + discardFailedBatch(); + throw new SQLRecoverableException(ex.getMessage(), "08003", ex); + } written += pending; pending = 0; statement.clearBatch(); } + /** + * A failed batch is rolled back and dropped, so close() neither sends it again nor counts it: + * committing an aborted transaction rolls it back without an error on some drivers. + */ + private void discardFailedBatch() + { + pending = 0; + try { + statement.clearBatch(); + } + catch (SQLException ex) { + log.debug("Clear failed batch: {}", ex.getMessage()); + } + if (transactional) { + try { + connection.rollback(); + } + catch (SQLException ex) { + log.warn("Rollback failed batch: {}", ex.getMessage()); + } + } + } + private void safeClose() { try { @@ -277,23 +384,45 @@ private void safeClose() } catch (SQLException ignore) { } - jdbcConnection.destroy(); + release.run(); + } + + private static boolean supportsTransactions(Connection connection) + { + try { + return connection.getMetaData() == null || connection.getMetaData().supportsTransactions(); + } + catch (SQLException | RuntimeException ex) { + // Unknown: keep the transactional path, which still degrades if commit() is unsupported. + return true; + } + } + } + + /** The target's identifier quote: a double quote on PostgreSQL and most engines, a backtick on MySQL and ClickHouse. */ + private static String identifierQuote(Connection connection) + { + try { + return connection.getMetaData().getIdentifierQuoteString().trim(); + } + catch (SQLException | RuntimeException ex) { + return "`"; } } - private static String buildInsertTemplate(String database, String table, List columns) + private static String buildInsertTemplate(String quote, String database, String table, List columns) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO "); if (database != null && !database.isEmpty()) { - sb.append('`').append(database).append("`."); + sb.append(quote).append(database).append(quote).append('.'); } - sb.append('`').append(table).append("` ("); + sb.append(quote).append(table).append(quote).append(" ("); for (int i = 0; i < columns.size(); i++) { if (i > 0) { sb.append(", "); } - sb.append('`').append(columns.get(i)).append('`'); + sb.append(quote).append(columns.get(i)).append(quote); } sb.append(") VALUES ("); for (int i = 0; i < columns.size(); i++) { diff --git a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java index 00ad601170..1d4bc96b6d 100644 --- a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java +++ b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java @@ -5,8 +5,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.devlive.metaforge.common.view.EntityView; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.security.CodeSource; import java.util.LinkedHashSet; import java.util.Set; @@ -43,6 +48,9 @@ public boolean supports(String capability) return capabilities.contains(capability); } + /** The declaration file in each plugin's jar, in the plugin package directory. */ + public static final String FILE = "org/devlive/metaforge/plugin/capabilities.json"; + /** A source without management: no workspace, no management entries. */ public static ManagementCapabilities none() { @@ -72,20 +80,47 @@ protected ManagementCapabilities computeValue(Class type) private static ManagementCapabilities load(Class service) { - // A service extending another plugin's service inherits its declaration unless it has its own. - Class owner = service; - while (owner != null && owner != Object.class && owner.getResource(owner.getSimpleName() + ".capabilities.json") == null) { - owner = owner.getSuperclass(); + // A service extending another plugin's service inherits its declaration unless its own jar has one. + for (Class owner = service; owner != null && owner != Object.class; owner = owner.getSuperclass()) { + URL declaration = declaration(owner); + if (declaration == null) { + continue; + } + try { + URLConnection connection = declaration.openConnection(); + // Plugins are reloaded from their jars; a cached jar would stay open and stale. + connection.setUseCaches(false); + try (InputStream stream = connection.getInputStream()) { + return read(declaration.toString(), stream); + } + } + catch (FileNotFoundException error) { + continue; + } + catch (IOException error) { + throw new IllegalStateException("Reading " + declaration + " failed", error); + } } - if (owner == null || owner == Object.class) { - return none(); + return none(); + } + + /** + * The capabilities.json of the plugin package in the jar or classes directory the class comes + * from. Every plugin puts it at the same path, so it is read from the class's own code source: + * a lookup by name would find whichever plugin comes first when several share a class path. + */ + private static URL declaration(Class type) + { + CodeSource source = type.getProtectionDomain() == null ? null : type.getProtectionDomain().getCodeSource(); + if (source == null || source.getLocation() == null) { + return null; } - String resource = owner.getSimpleName() + ".capabilities.json"; - try (InputStream stream = owner.getResourceAsStream(resource)) { - return stream == null ? none() : read(resource, stream); + try { + String location = source.getLocation().toString(); + return location.endsWith(".jar") ? new URL("jar:" + location + "!/" + FILE) : new URL(source.getLocation(), FILE); } - catch (IOException error) { - throw new IllegalStateException("Reading " + resource + " failed", error); + catch (MalformedURLException error) { + return null; } } diff --git a/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java index 5353f56c36..2484879f5e 100644 --- a/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java +++ b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java @@ -1,12 +1,18 @@ package org.devlive.metaforge.spi; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.devlive.metaforge.spi.model.ManagementCapabilities; import org.devlive.metaforge.spi.model.ManagementCapability; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -14,59 +20,107 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Objects.requireNonNull; + +@SuppressFBWarnings(value = {"DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED"}, justification = "Tests load plugin classes from throwaway directories") public class ManagementCapabilitiesTest { - static class DeclaringService + public static class DeclaringService { } - static class InheritingService + public static class InheritingService extends DeclaringService { } - static class SilentService + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + /** A classes directory holding the given class and, when given, its capabilities.json. */ + private Path plugin(Class type, String declaration) + throws IOException { + Path root = folder.newFolder().toPath(); + String name = type.getName().replace('.', '/') + ".class"; + Path target = root.resolve(name); + Files.createDirectories(requireNonNull(target.getParent())); + try (InputStream in = type.getClassLoader().getResourceAsStream(name)) { + Files.write(target, in.readAllBytes()); + } + if (declaration != null) { + Path file = root.resolve(ManagementCapabilities.FILE); + Files.createDirectories(requireNonNull(file.getParent())); + Files.writeString(file, declaration, StandardCharsets.UTF_8); + } + return root; } - static class BrokenService + private static Class load(Path root, ClassLoader parent, Class type) + throws Exception { + return Class.forName(type.getName(), false, new URLClassLoader(new URL[] {root.toUri().toURL()}, parent)); } + private static final String DECLARED = "{\"kind\": \"relational\", \"capabilities\": [\"query:execute\", \"database:list\", \"table:list\", \"column:read\", \"tab:structure\", \"tab:sql\"]}"; + @Test - public void aPluginDeclaresItsCapabilitiesNextToItsService() + public void aPluginDeclaresItsCapabilitiesInItsJar() + throws Exception { - ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(DeclaringService.class); + Class service = load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class); + ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(service); Assert.assertEquals("relational", capabilities.getKind()); Assert.assertTrue(capabilities.supports("tab:structure")); Assert.assertFalse(capabilities.supports("table:create")); } @Test - public void aSubclassWithoutItsOwnFileUsesItsParentsDeclaration() + public void pluginsSharingAPackageReadOnlyTheirOwnFile() + throws Exception { - Assert.assertEquals(ManagementCapabilities.declaredBy(DeclaringService.class).getCapabilities(), - ManagementCapabilities.declaredBy(InheritingService.class).getCapabilities()); + Class declaring = load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Class other = load(plugin(DeclaringService.class, "{\"kind\": \"key-value\", \"capabilities\": [\"query:execute\", \"tab:sql\"]}"), + ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Assert.assertEquals("relational", ManagementCapabilities.declaredBy(declaring).getKind()); + Assert.assertEquals("key-value", ManagementCapabilities.declaredBy(other).getKind()); + } + + @Test + public void aSubclassWhoseJarHasNoFileUsesItsParentsDeclaration() + throws Exception + { + Path parentRoot = plugin(DeclaringService.class, DECLARED); + URLClassLoader parent = new URLClassLoader(new URL[] {parentRoot.toUri().toURL()}, ClassLoader.getPlatformClassLoader()); + Class inheriting = load(plugin(InheritingService.class, null), parent, InheritingService.class); + Assert.assertEquals(parent, inheriting.getSuperclass().getClassLoader()); + Assert.assertTrue(ManagementCapabilities.declaredBy(inheriting).supports("table:list")); } @Test public void noDeclarationMeansNoManagement() + throws Exception { - ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(SilentService.class); + ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(load(plugin(DeclaringService.class, null), ClassLoader.getPlatformClassLoader(), DeclaringService.class)); Assert.assertEquals("unknown", capabilities.getKind()); Assert.assertTrue(capabilities.getCapabilities().isEmpty()); } - @Test(expected = IllegalStateException.class) + @Test public void anUnknownCapabilityIsAPluginError() + throws Exception { - ManagementCapabilities.declaredBy(BrokenService.class); + Class broken = load(plugin(DeclaringService.class, "{\"kind\": \"relational\", \"capabilities\": [\"query:execute\", \"table:teleport\"]}"), + ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Assert.assertThrows(IllegalStateException.class, () -> ManagementCapabilities.declaredBy(broken)); } @Test(expected = UnsupportedOperationException.class) public void callersCannotMutateAPluginCapabilitySet() + throws Exception { - ManagementCapabilities.declaredBy(DeclaringService.class).getCapabilities().clear(); + ManagementCapabilities.declaredBy(load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class)) + .getCapabilities().clear(); } @Test @@ -75,30 +129,31 @@ public void everyCapabilityKeyIsNamespaced() ManagementCapability.keys().forEach(key -> Assert.assertTrue(key, key.matches("[a-z-]+:[a-z-]+"))); } - /** Every plugin of the repository ships a valid declaration named after an existing service class. */ + /** Every plugin of the repository ships one valid capabilities.json in the plugin package of its resources. */ @Test public void everyPluginDeclarationIsValid() throws IOException { Path plugins = Paths.get("../../plugin").toAbsolutePath().normalize(); - Assert.assertTrue(plugins.toString(), Files.isDirectory(plugins)); - List files; - try (Stream walk = Files.walk(plugins)) { - files = walk.filter(path -> path.toString().endsWith(".capabilities.json") && path.toString().contains("src/main/resources")) + List modules; + try (Stream list = Files.list(plugins)) { + modules = list.filter(path -> path.getFileName().toString().startsWith("metaforge-plugin-") && Files.exists(path.resolve("pom.xml"))) .collect(Collectors.toList()); } - Assert.assertFalse(files.isEmpty()); - for (Path file : files) { + Assert.assertFalse(modules.isEmpty()); + for (Path module : modules) { + Path file = module.resolve("src/main/resources").resolve(ManagementCapabilities.FILE); + Assert.assertTrue(module.getFileName() + " has no " + ManagementCapabilities.FILE, Files.exists(file)); try (InputStream stream = Files.newInputStream(file)) { ManagementCapabilities capabilities = ManagementCapabilities.read(file.toString(), stream); if (!capabilities.getCapabilities().isEmpty()) { Assert.assertTrue(file + " can not run SQL", capabilities.supports("query:execute")); } } - String resource = file.toString(); - String source = resource.replace("src/main/resources", "src/main/java").replace(".capabilities.json", ".java"); - String kotlin = resource.replace("src/main/resources", "src/main/kotlin").replace(".capabilities.json", ".kt"); - Assert.assertTrue(file + " names no service class", Files.exists(Paths.get(source)) || Files.exists(Paths.get(kotlin))); + try (Stream walk = Files.walk(module.resolve("src/main/resources"))) { + Assert.assertEquals(module.getFileName() + " declares capabilities twice", 1, + walk.filter(path -> path.getFileName().toString().endsWith("capabilities.json")).count()); + } } } } diff --git a/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/adapter/JdbcBatchWriterTest.java b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/adapter/JdbcBatchWriterTest.java new file mode 100644 index 0000000000..5ef4e3ec52 --- /dev/null +++ b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/adapter/JdbcBatchWriterTest.java @@ -0,0 +1,158 @@ +package org.devlive.metaforge.spi.adapter; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** Batch writes follow the target's transaction support: analytical engines such as ClickHouse have none. */ +public class JdbcBatchWriterTest +{ + /** Records the calls a writer makes on a stubbed JDBC connection. */ + private static final class Driver + { + final List calls = new ArrayList<>(); + final AtomicInteger batches = new AtomicInteger(); + boolean transactions; + boolean commitUnsupported; + boolean batchFails; + boolean autoCommit = true; + + Connection connection() + { + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[] {PreparedStatement.class}, (proxy, method, args) -> { + if (method.getName().equals("executeBatch")) { + batches.incrementAndGet(); + if (batchFails) { + throw new SQLException("constraint violated"); + } + return new int[0]; + } + return null; + }); + DatabaseMetaData metadata = (DatabaseMetaData) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[] {DatabaseMetaData.class}, + (proxy, method, args) -> method.getName().equals("supportsTransactions") ? transactions : null); + return (Connection) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[] {Connection.class}, (proxy, method, args) -> { + calls.add(method.getName() + (args == null ? "" : List.of(args))); + switch (method.getName()) { + case "getMetaData": + return metadata; + case "getAutoCommit": + return autoCommit; + case "setAutoCommit": + autoCommit = (Boolean) args[0]; + return null; + case "prepareStatement": + return statement; + case "commit": + if (commitUnsupported) { + throw new SQLFeatureNotSupportedException("commit() is not supported."); + } + return null; + default: + return null; + } + }); + } + } + + private static void write(BatchWriter writer, int rows) + { + for (int index = 0; index < rows; index++) { + writer.addRow(List.of(index)); + } + writer.close(); + } + + @Test + public void targetsWithoutTransactionsWriteWithAutoCommit() + { + Driver driver = new Driver(); + AtomicBoolean released = new AtomicBoolean(); + BatchWriter writer = JdbcStreamAdapter.batchWriter(driver.connection(), () -> released.set(true), "db", "events", List.of("id"), 2); + write(writer, 5); + Assert.assertEquals(5, writer.writtenCount()); + Assert.assertEquals(3, driver.batches.get()); + Assert.assertFalse(driver.calls.contains("commit")); + Assert.assertTrue(driver.calls.stream().noneMatch(call -> call.startsWith("setAutoCommit"))); + Assert.assertTrue(released.get()); + } + + @Test + public void transactionalTargetsCommitEachBatchAndRestoreAutoCommit() + { + Driver driver = new Driver(); + driver.transactions = true; + BatchWriter writer = JdbcStreamAdapter.batchWriter(driver.connection(), () -> { }, "db", "events", List.of("id"), 2); + write(writer, 3); + Assert.assertEquals(3, writer.writtenCount()); + Assert.assertEquals(2, driver.calls.stream().filter("commit"::equals).count()); + Assert.assertTrue(driver.calls.contains("setAutoCommit[false]")); + Assert.assertTrue(driver.autoCommit); + } + + @Test + public void aFailedBatchIsRolledBackNotCountedAndNotSentAgainOnClose() + { + Driver driver = new Driver(); + driver.transactions = true; + AtomicBoolean released = new AtomicBoolean(); + BatchWriter writer = JdbcStreamAdapter.batchWriter(driver.connection(), () -> released.set(true), "db", "events", List.of("id"), 2); + writer.addRow(List.of(1)); + writer.addRow(List.of(2)); + Assert.assertEquals(2, writer.writtenCount()); + driver.batchFails = true; + writer.addRow(List.of(3)); + Assert.assertThrows(IllegalStateException.class, () -> writer.addRow(List.of(4))); + Assert.assertEquals(2, writer.writtenCount()); + Assert.assertTrue(driver.calls.contains("rollback")); + Assert.assertEquals(2, driver.batches.get()); + writer.close(); + Assert.assertEquals("close does not retry the failed batch", 2, driver.batches.get()); + Assert.assertEquals(2, writer.writtenCount()); + Assert.assertTrue(released.get()); + } + + @Test + public void flushCommitsAPartialBatchAndDoesNothingWhenNoneIsPending() + { + Driver driver = new Driver(); + driver.transactions = true; + BatchWriter writer = JdbcStreamAdapter.batchWriter(driver.connection(), () -> { }, "db", "events", List.of("id"), 10); + writer.addRow(List.of(1)); + writer.addRow(List.of(2)); + Assert.assertEquals(0, writer.writtenCount()); + writer.flush(); + Assert.assertEquals(2, writer.writtenCount()); + Assert.assertEquals(1, driver.batches.get()); + writer.flush(); + Assert.assertEquals(1, driver.batches.get()); + writer.close(); + Assert.assertEquals(1, driver.batches.get()); + Assert.assertEquals(2, writer.writtenCount()); + } + + @Test + public void driversThatRejectCommitKeepWriting() + { + Driver driver = new Driver(); + driver.transactions = true; + driver.commitUnsupported = true; + BatchWriter writer = JdbcStreamAdapter.batchWriter(driver.connection(), () -> { }, "db", "events", List.of("id"), 2); + write(writer, 5); + Assert.assertEquals(5, writer.writtenCount()); + Assert.assertEquals(3, driver.batches.get()); + // The first unsupported commit switches the writer to auto-commit; it is not retried. + Assert.assertEquals(1, driver.calls.stream().filter("commit"::equals).count()); + Assert.assertTrue(driver.autoCommit); + } +} diff --git a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json b/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json deleted file mode 100644 index 886b369cd3..0000000000 --- a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "kind": "relational", - "capabilities": ["query:execute", "table:teleport"] -} diff --git a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json b/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json deleted file mode 100644 index f3410db022..0000000000 --- a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "kind": "relational", - "capabilities": ["query:execute", "database:list", "table:list", "column:read", "tab:structure", "tab:sql"] -} diff --git a/core/metaforge-ui/src/components/MfBadge.vue b/core/metaforge-ui/src/components/MfBadge.vue index cdbe0587d1..17c034d53c 100644 --- a/core/metaforge-ui/src/components/MfBadge.vue +++ b/core/metaforge-ui/src/components/MfBadge.vue @@ -1,5 +1,5 @@