Enhance export features, refactor capabilities, and improve auth pages

.github/workflows/ci.yml

@@ -83,7 +83,9 @@ jobs:
   tests:
     name: Backend tests
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    # The plugin e2e modules alone need well over half an hour: every scheduler-based
+    # engine (presto, trino) starts its own container per documented version.
+    timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
 
@@ -97,8 +99,19 @@ jobs:
       # The root pom has no Surefire pin; Maven's default (2.12.4) cannot run
       # JUnit 5 and silently executes zero tests. Invoke Surefire explicitly
       # until the root pom pins the plugin version.
+      #
+      # Install first (skipping test execution) so downstream modules resolve
+      # packaged artifacts: the shaded drivers only exist after the package
+      # phase, and a bare test-compile would leave their plugin tests with a
+      # driver-less classpath.
+      - name: Build and install all modules
+        run: >
+          ./mvnw -B clean install
+          -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip
+          -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm
+
       - name: Run the JUnit 5 suite
         run: >
-          ./mvnw -B test-compile org.apache.maven.plugins:maven-surefire-plugin:3.2.2:test
+          ./mvnw -B org.apache.maven.plugins:maven-surefire-plugin:3.2.2:test
           -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip
           -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm

.github/workflows/release.yml

@@ -109,46 +109,133 @@ jobs:
   maven:
     name: Publish to Maven Central
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 150
     needs: build
+    env:
+      # Publishing to Maven Central is permanent, so it runs last and only when
+      # its secrets exist; a half-configured repository still releases the rest.
+      CENTRAL_CONFIGURED: ${{ secrets.CENTRAL_USERNAME != '' && secrets.GPG_PRIVATE_KEY != '' }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
 
-      # Imports the GPG key and injects the metaforge-ossrh server
-      # credentials into the Maven settings. The gpg-passphrase input is the
-      # NAME of the env variable holding the passphrase, provided below.
-      - name: Set up JDK 21 with signing and repository credentials
+      # The distribution and the store archives (dist/plugins) are built once
+      # here; the publishing steps below reuse them, so a deploy that fails
+      # half-way leaves nothing behind and re-running finishes the release.
+      - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: 21
           cache: maven
-          gpg-private-key: ${{ secrets.GPG_SECRET }}
-          gpg-passphrase: GPG_PASSPHRASE
-          server-id: metaforge-ossrh
-          server-username: OSSRH_USERNAME
-          server-password: OSSRH_TOKEN
-
-      # Fails fast with a clear message when the key or passphrase is wrong,
-      # instead of stalling or failing halfway through the deploy.
-      - name: Verify the GPG signing key
+      - name: Build the distribution and the store archives
+        run: >
+          ./mvnw -B clean install
+          -Dmaven.test.skip=true -Dcheckstyle.skip -Dspotbugs.skip -Dgpg.skip
+          -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm
+
+      - uses: actions/setup-java@v4
+        if: env.CENTRAL_CONFIGURED == 'true'
+        with:
+          distribution: temurin
+          java-version: 21
+          cache: maven
+          # Publishing goes to Maven Central through the Central Portal with the
+          # server id "central"; the gpg-passphrase input is the NAME of the env
+          # variable holding the passphrase, provided below.
+          server-id: central
+          server-username: CENTRAL_USERNAME
+          server-password: CENTRAL_PASSWORD
+          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE
+      - name: Publish the Maven artifacts to Maven Central
+        if: env.CENTRAL_CONFIGURED == 'true'
         env:
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSWORD }}
+          CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }}
+          CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
         run: |
-          gpg --list-secret-keys --keyid-format long
-          echo 'metaforge signing check' | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --local-user "$(gpg --list-secret-keys --keyid-format long --with-colons | awk -F: '/^sec/{print $5; exit}')" -o /tmp/signing-check.sig --sign
-          rm -f /tmp/signing-check.sig
+          version="$(./mvnw -B -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive \
+            -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN exec:exec | tail -1)"
+          if curl --silent --fail --head --output /dev/null \
+              "https://repo1.maven.org/maven2/org/devlive/metaforge/metaforge/${version}/metaforge-${version}.pom"; then
+            echo "::notice::${version} is on Maven Central already"
+            exit 0
+          fi
+          ./mvnw --batch-mode --no-transfer-progress -Prelease,central -Dcentral.skip=false \
+            -Dmaven.test.skip=true -Dcheckstyle.skip -Dspotbugs.skip \
+            -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm deploy \
+            -pl '!test/metaforge-test-condor,!test/metaforge-test-convert,!test/metaforge-test-core,!test/metaforge-test-lib,!test/metaforge-test-executor,!test/metaforge-test-scheduler,!test/metaforge-test-fs,!test/metaforge-test-driver,!test/metaforge-test-parser,!test/metaforge-test-plugin'
+      - name: Skip Maven Central
+        if: env.CENTRAL_CONFIGURED != 'true'
+        run: echo "::notice::Maven Central skipped; set CENTRAL_USERNAME, CENTRAL_PASSWORD, GPG_PRIVATE_KEY and GPG_PASSPHRASE to publish there"
+
+      # The store catalogue and the statistics catalog are built from the very
+      # archives just deployed to Maven Central (dist/plugins holds copies), so
+      # one SHA-256 and size are valid for the Maven and CDN sources alike.
+      - name: Build the store catalogue and the statistics catalog
+        run: >
+          node configure/publish/build-metadata.mjs --dist dist/plugins --strict
+          --out dist/metadata.json --catalog dist/release-catalog.json
+          --commit "$GITHUB_SHA" --generation "$GITHUB_RUN_NUMBER" --released now
+
+      - name: Upload the catalogues
+        uses: actions/upload-artifact@v4
+        with:
+          name: store-catalogue
+          if-no-files-found: error
+          path: |
+            dist/metadata.json
+            dist/release-catalog.json
 
-      - name: Deploy to Maven Central
+      # Order: packages to the CDN -> catalog into ForgeStats -> public metadata.json.
+      # A failed step keeps the previous public catalogue, and re-running is safe.
+      - name: Publish the packages to the CDN
         env:
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }}
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSWORD }}
-        run: >
-          ./mvnw -B clean deploy -Pdeploy
-          -DskipTests -Dcheckstyle.skip -Dspotbugs.skip
-          -Dskip.pnpm -Dskip.npm -Dskip.installnodenpm
-          -Dgpg.passphrase="$GPG_PASSPHRASE"
-          -pl '!test/metaforge-test-condor,!test/metaforge-test-convert,!test/metaforge-test-core,!test/metaforge-test-lib,!test/metaforge-test-executor,!test/metaforge-test-scheduler,!test/metaforge-test-fs,!test/metaforge-test-driver,!test/metaforge-test-parser,!test/metaforge-test-plugin'
+          OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+          OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
+          OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }}
+          OSS_BUCKET: ${{ secrets.OSS_BUCKET }}
+        run: |
+          if [ -z "$OSS_ACCESS_KEY_ID" ]; then
+            echo '::warning::OSS secrets are not set; skipping the CDN upload (the store falls back to Maven Central)'
+            exit 0
+          fi
+          # Official ossutil 1.x installer (same tool as configure/publish/publish-oss-artifact.sh).
+          curl -fsSL https://gosspublic.alicdn.com/ossutil/install.sh | sudo bash
+          oss() { ossutil -e "$OSS_ENDPOINT" -i "$OSS_ACCESS_KEY_ID" -k "$OSS_ACCESS_KEY_SECRET" "$@"; }
+          jq -r '.plugins[] | .sources.cdn' dist/release-catalog.json | while read -r path; do
+            file="dist/plugins/$(basename "$path")"
+            oss cp "$file" "oss://$OSS_BUCKET/applications/metaforge/plugins/$path" -f
+          done
+
+      - name: Import the statistics catalog into ForgeStats
+        env:
+          STATS_SSH_KEY: ${{ secrets.STATS_SSH_KEY }}
+          STATS_SSH_TARGET: ${{ secrets.STATS_SSH_TARGET }}
+        run: |
+          if [ -z "$STATS_SSH_KEY" ] || [ -z "$STATS_SSH_TARGET" ]; then
+            echo '::warning::ForgeStats secrets are not set; downloads of this release are not counted'
+            exit 0
+          fi
+          umask 077
+          printf '%s\n' "$STATS_SSH_KEY" > "$RUNNER_TEMP/stats-key"
+          digest=$(sha256sum dist/release-catalog.json | cut -d ' ' -f 1)
+          ssh -T -i "$RUNNER_TEMP/stats-key" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new \
+            "$STATS_SSH_TARGET" "sudo -n /usr/local/sbin/forgestats-admin catalog metaforge $digest" < dist/release-catalog.json
+          rm -f "$RUNNER_TEMP/stats-key"
+
+      - name: Publish the store catalogue
+        env:
+          OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+          OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
+          OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT }}
+          OSS_BUCKET: ${{ secrets.OSS_BUCKET }}
+        run: |
+          if [ -z "$OSS_ACCESS_KEY_ID" ]; then
+            echo '::warning::OSS secrets are not set; publish dist/metadata.json from the store-catalogue artifact manually'
+            exit 0
+          fi
+          ossutil -e "$OSS_ENDPOINT" -i "$OSS_ACCESS_KEY_ID" -k "$OSS_ACCESS_KEY_SECRET" \
+            cp dist/metadata.json "oss://$OSS_BUCKET/applications/metaforge/metadata/metadata.json" -f

README.md

@@ -4,16 +4,14 @@
 
 # MetaForge
 
----
+**Integrated software for data transformation, integration, and visualization**
 
-![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fmetaforge.git&countColor=%23263759&style=flat&labelStyle=none)
-[![](https://tokei.rs/b1/github/EdurtIO/metaforge)](https://github.com/EdurtIO/metaforge)
-![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg)
+---
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-![GitHub Release Date](https://img.shields.io/github/release-date/EdurtIO/metaforge?style=flat-square)
+![version](https://img.shields.io/github/v/release/EdurtIO/metaforge.svg)
 
-![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge)
+[![Docker Automated build](https://img.shields.io/docker/automated/devliveorg/metaforge)](https://hub.docker.com/r/devliveorg/metaforge)
 ![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/devliveorg/metaforge?style=flat-square)
 ![Docker Pulls](https://img.shields.io/docker/pulls/devliveorg/metaforge?style=flat-square)
 
@@ -23,28 +21,118 @@
 
 </div>
 
-### What is metaforge?
+## What is MetaForge?
+
+---
+
+MetaForge is integrated software for data transformation, integration, and visualization. It supports a wide variety of data sources — relational databases, NoSQL databases, big data engines, message queues and file systems — and lets you manage multiple data sources, query and transform data, build datasets, and create charts and monitoring views.
+
+## Features
 
 ---
 
-MetaForge is integrated software for data transformation, integration, and visualization. Support a variety of data sources, file types, big data related database, relational database, NoSQL database, etc. Through the software can realize the management of multiple data sources, the data under the source of various operations conversion, making data charts, monitoring data sources and other functions.
+- **Data source management** — connect, test and manage 50+ data sources from a single console
+- **SQL query console** — IDE-style editor with syntax highlighting, formatting, execution history and export
+- **Datasets** — build reusable datasets from tables or ad-hoc queries, preview data and define column metadata
+- **Visualization** — drag-and-drop dashboards with tables, pies and more chart types
+- **Reports** — schedule and share reports
+- **Workflows** — chain data jobs into repeatable workflows
+- **File storage** — read and write files through Aliyun OSS, Amazon S3, HDFS, local storage and more
+- **Notifications** — DingTalk, email and other channels for job and system events
+- **AI assistant** — chat with your data through configured AI providers
+- **Plugin architecture** — every connector, converter and storage backend is a hot-loadable plugin
+- **Administration** — users, roles, menus and runtime configuration with fine-grained permissions
+
+## Requirements
 
-| Username  | Password    |
-|:---------:|:------------|
-| `metaforge` | `123456789` |
-|  `admin`  | `12345678`  |
+---
+
+| Dependency | Version                                      |
+|:-----------|:---------------------------------------------|
+| JDK        | 21                                           |
+| Maven      | 3.9+ (wrapper `./mvnw` included)             |
+| Node.js    | 20                                           |
+| pnpm       | 9.13.2                                       |
 
-### Require
+## Getting Started
 
 ---
 
-> Must-read for users: Be sure to execute the following command after cloning the code locally
+> Must-read for users: be sure to execute the following command after cloning the code locally
 
 ```bash
 cp configure/git-hook/* .git/hooks
 chmod 700 .git/hooks/*
 ```
 
+### Run the backend (development)
+
+Run the main class `org.devlive.metaforge.server.MetaForge` (module `core/metaforge-server`) from your IDE.
+The server starts on [http://localhost:9096](http://localhost:9096). On first launch the setup wizard guides you through
+selecting the metadata database (embedded H2 by default, or MySQL / PostgreSQL / SQL Server / Oracle) and creating the
+administrator account.
+
+### Run the frontend (development)
+
+```bash
+pnpm install
+pnpm --filter metaforge-web dev
+```
+
+The web console starts on [http://localhost:3001](http://localhost:3001) and proxies API calls to the backend.
+
+### Build for production
+
+```bash
+# Frontend
+pnpm --filter metaforge-web build
+
+# Backend + distribution package (the server serves the built frontend
+# from core/metaforge-web/dist; the tarball lands in dist/)
+./mvnw -T1C clean package -DskipTests
+```
+
+## Docker
+
+---
+
+```bash
+docker pull devliveorg/metaforge:latest
+docker run -d --name metaforge -p 9096:9096 devliveorg/metaforge:latest
+```
+
+Or unpack the distribution built locally (`dist/metaforge-server-2026.0.0-bin.tar.gz`) and run `bin/startup.sh`.
+
+## Project Structure
+
+---
+
+```
+metaforge
+├── client                        # Command line client
+├── configure                     # Default configuration and plugin manifests
+├── core
+│   ├── metaforge-captcha         # Captcha generation
+│   ├── metaforge-common          # Shared utilities and enums
+│   ├── metaforge-condor          # Scheduling integration
+│   ├── metaforge-server          # Spring Boot server (REST API entry point)
+│   ├── metaforge-service         # Domain services and persistence
+│   ├── metaforge-security        # Security configuration
+│   ├── metaforge-sql             # SQL statement builders
+│   ├── metaforge-spi             # Plugin SPI contracts
+│   ├── metaforge-web             # Vue 3 + Tailwind web console
+├── convert                       # Data converters (JSON, CSV, XML, ...)
+├── driver                        # Database drivers (Redis, MongoDB, ...)
+├── executor                      # Execution engines
+├── fs                            # File storage plugins (OSS, S3, local, ...)
+├── lib                           # Shared libraries (HTTP client, logging, shell)
+├── notify                        # Notification channels (DingTalk, ...)
+├── parser                        # SQL parsers (MySQL, Trino, ...)
+├── plugin                        # Data source plugins (50+ connectors)
+├── scheduler                     # Job scheduling
+└── shaded                        # Shaded third-party integrations
+```
+
 ## Supported Connectors
 
 ---
@@ -105,17 +193,11 @@ Here are some of the major database solutions that are supported:
  <a href="https://dolphindb.com/"><img src="docs/assets/plugin/dolphindb.svg" alt="DolphinDB" height="60"/></a>
 </div>
 
-## System architecture
-
----
-
-<img src="docs/assets/architecture.jpg" width="100%" />
-
-## Stargazers over time
+## Documentation
 
 ---
 
-[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline)
+[Extended documentation for MetaForge](https://metaforge.devlive.org)
 
 ## Join Us
 
@@ -131,11 +213,11 @@ Please visit [Issues 950](https://github.com/devlive-community/metaforge/issues/
 [![App Store](https://img.shields.io/badge/App%20Store-Rainbond-brightgreen?style=flat-square)](https://www.rainbond.com/)
 [![Vue 3](https://img.shields.io/badge/UI-Vue%203%20%2B%20Tailwind-brightgreen?style=flat-square)](https://vuejs.org/)
 
-## Installation and Configuration
+## Stargazers over time
 
 ---
 
-[Extended documentation for MetaForge](https://metaforge.devlive.org)
+[![Star History Chart](https://star-history.dera.page/svg?repos=devlive-community/metaforge&type=Timeline)](https://star-history.dera.page/#devlive-community/metaforge&Timeline)
 
 ## Contributors