From 64ed63e369955ddbe6501cffe722db8818641453 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 21:39:38 -0400 Subject: [PATCH 001/232] fix(er): span table card headers across the card The tooltip around a database ER card's header kept the header as wide as the table name; it now spans the card, a long name ends in an ellipsis and the tooltip shows the full name with the table comment. --- .../src/views/pages/tools/DatabaseEr.vue | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/metaforge-web/src/views/pages/tools/DatabaseEr.vue b/core/metaforge-web/src/views/pages/tools/DatabaseEr.vue index 76f33ff676..7acf3beec6 100644 --- a/core/metaforge-web/src/views/pages/tools/DatabaseEr.vue +++ b/core/metaforge-web/src/views/pages/tools/DatabaseEr.vue @@ -34,11 +34,16 @@
- - +
From f6bc3ec6a3fabd4c5586a516a4e78d9580724395 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 21:47:22 -0400 Subject: [PATCH 002/232] refactor(plugin): name every capability declaration capabilities.json Each plugin declares its capabilities in src/main/resources/org/devlive/metaforge/plugin/ capabilities.json instead of .capabilities.json. As every plugin uses the same path, the SPI reads the file from the jar or classes directory of the plugin's service class rather than by resource name, so plugins on one class path never read each other's; a subclass whose jar has no file still uses its parent's. The tests load classes from throwaway directories, and the plugin guide shows the path. --- .../spi/model/ManagementCapabilities.java | 57 ++++++++-- .../spi/ManagementCapabilitiesTest.java | 103 ++++++++++++++---- .../spi/BrokenService.capabilities.json | 4 - .../spi/DeclaringService.capabilities.json | 4 - docs/content/developer/plugin/java/home.en.md | 4 +- docs/content/developer/plugin/java/home.md | 4 +- ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 ...ce.capabilities.json => capabilities.json} | 0 57 files changed, 129 insertions(+), 47 deletions(-) delete mode 100644 core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json delete mode 100644 core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json rename plugin/metaforge-plugin-alioss/src/main/resources/org/devlive/metaforge/plugin/{AliOssService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-cassandra/src/main/resources/org/devlive/metaforge/plugin/{CassandraService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-ceresdb/src/main/resources/org/devlive/metaforge/plugin/{CeresDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-clickhouse/src/main/resources/org/devlive/metaforge/plugin/{ClickHouseService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-cratedb/src/main/resources/org/devlive/metaforge/plugin/{CrateDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-db2/src/main/resources/org/devlive/metaforge/plugin/{Db2Service.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-dm/src/main/resources/org/devlive/metaforge/plugin/{DmService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-dolphindb/src/main/resources/org/devlive/metaforge/plugin/{dolphindb/DolphinDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-doris/src/main/resources/org/devlive/metaforge/plugin/{DorisService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-dremio/src/main/resources/org/devlive/metaforge/plugin/{DremioService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-druid/src/main/resources/org/devlive/metaforge/plugin/{DruidService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-duckdb/src/main/resources/org/devlive/metaforge/plugin/{DuckDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-elasticsearch-8x/src/main/resources/org/devlive/metaforge/plugin/{ElasticSearch8xService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-greptimedb/src/main/resources/org/devlive/metaforge/plugin/{GreptimeDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-h2-tcp/src/main/resources/org/devlive/metaforge/plugin/{H2TCPService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-h2/src/main/resources/org/devlive/metaforge/plugin/{H2Service.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-hdfs/src/main/resources/org/devlive/metaforge/plugin/{HdfsService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-hive-2x/src/main/resources/org/devlive/metaforge/plugin/{Hive2xService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-hologres/src/main/resources/org/devlive/metaforge/plugin/{HologresService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-ignite/src/main/resources/org/devlive/metaforge/plugin/{IgniteService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-impala/src/main/resources/org/devlive/metaforge/plugin/{ImpalaService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-influxdb/src/main/resources/org/devlive/metaforge/plugin/{influxdb/InfluxdbService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-iotdb/src/main/resources/org/devlive/metaforge/plugin/{IoTDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-kafka/src/main/resources/org/devlive/metaforge/plugin/{KafkaService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-kylin/src/main/resources/org/devlive/metaforge/plugin/{KylinService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-kyuubi/src/main/resources/org/devlive/metaforge/plugin/{KyuubiService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-matrixone/src/main/resources/org/devlive/metaforge/plugin/{MatrixOneService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-monetdb/src/main/resources/org/devlive/metaforge/plugin/{MonetDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-mongo-atlas/src/main/resources/org/devlive/metaforge/plugin/{MongoAtlasService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-mongo-community/src/main/resources/org/devlive/metaforge/plugin/{MongoCommunityService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-mysql/src/main/resources/org/devlive/metaforge/plugin/{MySQLService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-neo4j/src/main/resources/org/devlive/metaforge/plugin/{Neo4jService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-oceanbase/src/main/resources/org/devlive/metaforge/plugin/{OceanBaseService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-oracle/src/main/resources/org/devlive/metaforge/plugin/{OracleService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-paradedb/src/main/resources/org/devlive/metaforge/plugin/{ParadeDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-phoenix/src/main/resources/org/devlive/metaforge/plugin/{PhoenixService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-pinot/src/main/resources/org/devlive/metaforge/plugin/{PinotService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-postgresql/src/main/resources/org/devlive/metaforge/plugin/{PostgreSQLService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-presto/src/main/resources/org/devlive/metaforge/plugin/{PrestoService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-questdb/src/main/resources/org/devlive/metaforge/plugin/{QuestDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-redis/src/main/resources/org/devlive/metaforge/plugin/{RedisService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-scylladb/src/main/resources/org/devlive/metaforge/plugin/{ScyllaDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-snowflake/src/main/resources/org/devlive/metaforge/plugin/{SnowflakeService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-solr/src/main/resources/org/devlive/metaforge/plugin/{solr/SolrService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-sqlserver/src/main/resources/org/devlive/metaforge/plugin/{SqlServerService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-starrocks/src/main/resources/org/devlive/metaforge/plugin/{StarRocksService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-tdengine/src/main/resources/org/devlive/metaforge/plugin/{TDengineService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-timescale/src/main/resources/org/devlive/metaforge/plugin/{TimescaleService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-trino/src/main/resources/org/devlive/metaforge/plugin/{TrinoService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-ydb/src/main/resources/org/devlive/metaforge/plugin/{YDBService.capabilities.json => capabilities.json} (100%) rename plugin/metaforge-plugin-zookeeper/src/main/resources/org/devlive/metaforge/plugin/{ZookeeperService.capabilities.json => capabilities.json} (100%) diff --git a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java index 00ad601170..1d4bc96b6d 100644 --- a/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java +++ b/core/metaforge-spi/src/main/java/org/devlive/metaforge/spi/model/ManagementCapabilities.java @@ -5,8 +5,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.devlive.metaforge.common.view.EntityView; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.security.CodeSource; import java.util.LinkedHashSet; import java.util.Set; @@ -43,6 +48,9 @@ public boolean supports(String capability) return capabilities.contains(capability); } + /** The declaration file in each plugin's jar, in the plugin package directory. */ + public static final String FILE = "org/devlive/metaforge/plugin/capabilities.json"; + /** A source without management: no workspace, no management entries. */ public static ManagementCapabilities none() { @@ -72,20 +80,47 @@ protected ManagementCapabilities computeValue(Class type) private static ManagementCapabilities load(Class service) { - // A service extending another plugin's service inherits its declaration unless it has its own. - Class owner = service; - while (owner != null && owner != Object.class && owner.getResource(owner.getSimpleName() + ".capabilities.json") == null) { - owner = owner.getSuperclass(); + // A service extending another plugin's service inherits its declaration unless its own jar has one. + for (Class owner = service; owner != null && owner != Object.class; owner = owner.getSuperclass()) { + URL declaration = declaration(owner); + if (declaration == null) { + continue; + } + try { + URLConnection connection = declaration.openConnection(); + // Plugins are reloaded from their jars; a cached jar would stay open and stale. + connection.setUseCaches(false); + try (InputStream stream = connection.getInputStream()) { + return read(declaration.toString(), stream); + } + } + catch (FileNotFoundException error) { + continue; + } + catch (IOException error) { + throw new IllegalStateException("Reading " + declaration + " failed", error); + } } - if (owner == null || owner == Object.class) { - return none(); + return none(); + } + + /** + * The capabilities.json of the plugin package in the jar or classes directory the class comes + * from. Every plugin puts it at the same path, so it is read from the class's own code source: + * a lookup by name would find whichever plugin comes first when several share a class path. + */ + private static URL declaration(Class type) + { + CodeSource source = type.getProtectionDomain() == null ? null : type.getProtectionDomain().getCodeSource(); + if (source == null || source.getLocation() == null) { + return null; } - String resource = owner.getSimpleName() + ".capabilities.json"; - try (InputStream stream = owner.getResourceAsStream(resource)) { - return stream == null ? none() : read(resource, stream); + try { + String location = source.getLocation().toString(); + return location.endsWith(".jar") ? new URL("jar:" + location + "!/" + FILE) : new URL(source.getLocation(), FILE); } - catch (IOException error) { - throw new IllegalStateException("Reading " + resource + " failed", error); + catch (MalformedURLException error) { + return null; } } diff --git a/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java index 5353f56c36..2484879f5e 100644 --- a/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java +++ b/core/metaforge-spi/src/test/java/org/devlive/metaforge/spi/ManagementCapabilitiesTest.java @@ -1,12 +1,18 @@ package org.devlive.metaforge.spi; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.devlive.metaforge.spi.model.ManagementCapabilities; import org.devlive.metaforge.spi.model.ManagementCapability; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -14,59 +20,107 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Objects.requireNonNull; + +@SuppressFBWarnings(value = {"DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED"}, justification = "Tests load plugin classes from throwaway directories") public class ManagementCapabilitiesTest { - static class DeclaringService + public static class DeclaringService { } - static class InheritingService + public static class InheritingService extends DeclaringService { } - static class SilentService + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + /** A classes directory holding the given class and, when given, its capabilities.json. */ + private Path plugin(Class type, String declaration) + throws IOException { + Path root = folder.newFolder().toPath(); + String name = type.getName().replace('.', '/') + ".class"; + Path target = root.resolve(name); + Files.createDirectories(requireNonNull(target.getParent())); + try (InputStream in = type.getClassLoader().getResourceAsStream(name)) { + Files.write(target, in.readAllBytes()); + } + if (declaration != null) { + Path file = root.resolve(ManagementCapabilities.FILE); + Files.createDirectories(requireNonNull(file.getParent())); + Files.writeString(file, declaration, StandardCharsets.UTF_8); + } + return root; } - static class BrokenService + private static Class load(Path root, ClassLoader parent, Class type) + throws Exception { + return Class.forName(type.getName(), false, new URLClassLoader(new URL[] {root.toUri().toURL()}, parent)); } + private static final String DECLARED = "{\"kind\": \"relational\", \"capabilities\": [\"query:execute\", \"database:list\", \"table:list\", \"column:read\", \"tab:structure\", \"tab:sql\"]}"; + @Test - public void aPluginDeclaresItsCapabilitiesNextToItsService() + public void aPluginDeclaresItsCapabilitiesInItsJar() + throws Exception { - ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(DeclaringService.class); + Class service = load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class); + ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(service); Assert.assertEquals("relational", capabilities.getKind()); Assert.assertTrue(capabilities.supports("tab:structure")); Assert.assertFalse(capabilities.supports("table:create")); } @Test - public void aSubclassWithoutItsOwnFileUsesItsParentsDeclaration() + public void pluginsSharingAPackageReadOnlyTheirOwnFile() + throws Exception { - Assert.assertEquals(ManagementCapabilities.declaredBy(DeclaringService.class).getCapabilities(), - ManagementCapabilities.declaredBy(InheritingService.class).getCapabilities()); + Class declaring = load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Class other = load(plugin(DeclaringService.class, "{\"kind\": \"key-value\", \"capabilities\": [\"query:execute\", \"tab:sql\"]}"), + ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Assert.assertEquals("relational", ManagementCapabilities.declaredBy(declaring).getKind()); + Assert.assertEquals("key-value", ManagementCapabilities.declaredBy(other).getKind()); + } + + @Test + public void aSubclassWhoseJarHasNoFileUsesItsParentsDeclaration() + throws Exception + { + Path parentRoot = plugin(DeclaringService.class, DECLARED); + URLClassLoader parent = new URLClassLoader(new URL[] {parentRoot.toUri().toURL()}, ClassLoader.getPlatformClassLoader()); + Class inheriting = load(plugin(InheritingService.class, null), parent, InheritingService.class); + Assert.assertEquals(parent, inheriting.getSuperclass().getClassLoader()); + Assert.assertTrue(ManagementCapabilities.declaredBy(inheriting).supports("table:list")); } @Test public void noDeclarationMeansNoManagement() + throws Exception { - ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(SilentService.class); + ManagementCapabilities capabilities = ManagementCapabilities.declaredBy(load(plugin(DeclaringService.class, null), ClassLoader.getPlatformClassLoader(), DeclaringService.class)); Assert.assertEquals("unknown", capabilities.getKind()); Assert.assertTrue(capabilities.getCapabilities().isEmpty()); } - @Test(expected = IllegalStateException.class) + @Test public void anUnknownCapabilityIsAPluginError() + throws Exception { - ManagementCapabilities.declaredBy(BrokenService.class); + Class broken = load(plugin(DeclaringService.class, "{\"kind\": \"relational\", \"capabilities\": [\"query:execute\", \"table:teleport\"]}"), + ClassLoader.getPlatformClassLoader(), DeclaringService.class); + Assert.assertThrows(IllegalStateException.class, () -> ManagementCapabilities.declaredBy(broken)); } @Test(expected = UnsupportedOperationException.class) public void callersCannotMutateAPluginCapabilitySet() + throws Exception { - ManagementCapabilities.declaredBy(DeclaringService.class).getCapabilities().clear(); + ManagementCapabilities.declaredBy(load(plugin(DeclaringService.class, DECLARED), ClassLoader.getPlatformClassLoader(), DeclaringService.class)) + .getCapabilities().clear(); } @Test @@ -75,30 +129,31 @@ public void everyCapabilityKeyIsNamespaced() ManagementCapability.keys().forEach(key -> Assert.assertTrue(key, key.matches("[a-z-]+:[a-z-]+"))); } - /** Every plugin of the repository ships a valid declaration named after an existing service class. */ + /** Every plugin of the repository ships one valid capabilities.json in the plugin package of its resources. */ @Test public void everyPluginDeclarationIsValid() throws IOException { Path plugins = Paths.get("../../plugin").toAbsolutePath().normalize(); - Assert.assertTrue(plugins.toString(), Files.isDirectory(plugins)); - List files; - try (Stream walk = Files.walk(plugins)) { - files = walk.filter(path -> path.toString().endsWith(".capabilities.json") && path.toString().contains("src/main/resources")) + List modules; + try (Stream list = Files.list(plugins)) { + modules = list.filter(path -> path.getFileName().toString().startsWith("metaforge-plugin-") && Files.exists(path.resolve("pom.xml"))) .collect(Collectors.toList()); } - Assert.assertFalse(files.isEmpty()); - for (Path file : files) { + Assert.assertFalse(modules.isEmpty()); + for (Path module : modules) { + Path file = module.resolve("src/main/resources").resolve(ManagementCapabilities.FILE); + Assert.assertTrue(module.getFileName() + " has no " + ManagementCapabilities.FILE, Files.exists(file)); try (InputStream stream = Files.newInputStream(file)) { ManagementCapabilities capabilities = ManagementCapabilities.read(file.toString(), stream); if (!capabilities.getCapabilities().isEmpty()) { Assert.assertTrue(file + " can not run SQL", capabilities.supports("query:execute")); } } - String resource = file.toString(); - String source = resource.replace("src/main/resources", "src/main/java").replace(".capabilities.json", ".java"); - String kotlin = resource.replace("src/main/resources", "src/main/kotlin").replace(".capabilities.json", ".kt"); - Assert.assertTrue(file + " names no service class", Files.exists(Paths.get(source)) || Files.exists(Paths.get(kotlin))); + try (Stream walk = Files.walk(module.resolve("src/main/resources"))) { + Assert.assertEquals(module.getFileName() + " declares capabilities twice", 1, + walk.filter(path -> path.getFileName().toString().endsWith("capabilities.json")).count()); + } } } } diff --git a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json b/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json deleted file mode 100644 index 886b369cd3..0000000000 --- a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/BrokenService.capabilities.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "kind": "relational", - "capabilities": ["query:execute", "table:teleport"] -} diff --git a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json b/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json deleted file mode 100644 index f3410db022..0000000000 --- a/core/metaforge-spi/src/test/resources/org/devlive/metaforge/spi/DeclaringService.capabilities.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "kind": "relational", - "capabilities": ["query:execute", "database:list", "table:list", "column:read", "tab:structure", "tab:sql"] -} diff --git a/docs/content/developer/plugin/java/home.en.md b/docs/content/developer/plugin/java/home.en.md index 48056e3957..3a1716b1c7 100644 --- a/docs/content/developer/plugin/java/home.en.md +++ b/docs/content/developer/plugin/java/home.en.md @@ -264,7 +264,7 @@ The content of this file is the plugin loading module we defined. ### Management capabilities -Every page, tab, dialog, drawer, side panel and button of source management (console, data import, data export, data compare, ER diagram) is controlled by one management capability. A plugin declares the capabilities it implements in `.capabilities.json` next to its service class, for example `src/main/resources/org/devlive/metaforge/plugin/MySQLService.capabilities.json`: +Every page, tab, dialog, drawer, side panel and button of source management (console, data import, data export, data compare, ER diagram) is controlled by one management capability. A plugin declares the capabilities it implements in `src/main/resources/org/devlive/metaforge/plugin/capabilities.json` (every plugin uses this path; only the copy in the plugin's own jar is read): ```json { @@ -277,7 +277,7 @@ Every page, tab, dialog, drawer, side panel and button of source management (con - A plugin without a declaration offers no management: the source list hides its Manage entry and the related pages do not open. - Declare only what the plugin implements; the server refuses every operation the plugin did not declare. - Every plugin reaches its source through SQL, so a plugin that declares capabilities always declares `query:execute`. -- An unknown capability or kind fails when the plugin loads; a subclass without its own file uses its parent's declaration. +- An unknown capability or kind fails when the plugin loads; a subclass whose jar has no file uses its parent's declaration. - A new management feature first registers its capability in `ManagementCapability`, then plugins declare it, and every surface of the feature checks it. | Capability | Controls | diff --git a/docs/content/developer/plugin/java/home.md b/docs/content/developer/plugin/java/home.md index a5a54787de..b0b1e82415 100644 --- a/docs/content/developer/plugin/java/home.md +++ b/docs/content/developer/plugin/java/home.md @@ -265,7 +265,7 @@ org.devlive.metaforge.plugin.QuestDBPluginModule ### 管理能力声明 -数据源管理（控制台、数据导入、数据导出、数据对比、ER 图）中的每个页面、标签页、弹窗、抽屉、侧边栏和按钮都由一个管理能力控制。插件在服务类旁边的 `<服务类名>.capabilities.json` 中声明自己实现了哪些能力，例如 `src/main/resources/org/devlive/metaforge/plugin/MySQLService.capabilities.json`： +数据源管理（控制台、数据导入、数据导出、数据对比、ER 图）中的每个页面、标签页、弹窗、抽屉、侧边栏和按钮都由一个管理能力控制。插件在 `src/main/resources/org/devlive/metaforge/plugin/capabilities.json` 中声明自己实现了哪些能力（每个插件都放在这个路径，读取时只读插件自己 jar 里的这一份）： ```json { @@ -278,7 +278,7 @@ org.devlive.metaforge.plugin.QuestDBPluginModule - 没有声明文件的插件不提供数据管理：数据源列表隐藏「数据管理」入口，相关页面也无法打开。 - 声明的每个能力都必须真正实现；服务端同样按声明拒绝未声明的操作。 - 所有插件都通过 SQL 访问数据源，因此声明了能力的插件必须包含 `query:execute`。 -- 声明未知的能力或 `kind` 会在加载插件时报错；子类没有自己的声明文件时沿用父类的声明。 +- 声明未知的能力或 `kind` 会在加载插件时报错；子类所在的 jar 没有声明文件时沿用父类 jar 里的声明。 - 新增管理功能时，先在 `ManagementCapability` 中登记能力，再在需要的插件中声明，并让该功能的每个界面入口都检查这个能力。 | 能力 | 控制的界面 | diff --git a/plugin/metaforge-plugin-alioss/src/main/resources/org/devlive/metaforge/plugin/AliOssService.capabilities.json b/plugin/metaforge-plugin-alioss/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-alioss/src/main/resources/org/devlive/metaforge/plugin/AliOssService.capabilities.json rename to plugin/metaforge-plugin-alioss/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-cassandra/src/main/resources/org/devlive/metaforge/plugin/CassandraService.capabilities.json b/plugin/metaforge-plugin-cassandra/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-cassandra/src/main/resources/org/devlive/metaforge/plugin/CassandraService.capabilities.json rename to plugin/metaforge-plugin-cassandra/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-ceresdb/src/main/resources/org/devlive/metaforge/plugin/CeresDBService.capabilities.json b/plugin/metaforge-plugin-ceresdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-ceresdb/src/main/resources/org/devlive/metaforge/plugin/CeresDBService.capabilities.json rename to plugin/metaforge-plugin-ceresdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-clickhouse/src/main/resources/org/devlive/metaforge/plugin/ClickHouseService.capabilities.json b/plugin/metaforge-plugin-clickhouse/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-clickhouse/src/main/resources/org/devlive/metaforge/plugin/ClickHouseService.capabilities.json rename to plugin/metaforge-plugin-clickhouse/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-cratedb/src/main/resources/org/devlive/metaforge/plugin/CrateDBService.capabilities.json b/plugin/metaforge-plugin-cratedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-cratedb/src/main/resources/org/devlive/metaforge/plugin/CrateDBService.capabilities.json rename to plugin/metaforge-plugin-cratedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-db2/src/main/resources/org/devlive/metaforge/plugin/Db2Service.capabilities.json b/plugin/metaforge-plugin-db2/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-db2/src/main/resources/org/devlive/metaforge/plugin/Db2Service.capabilities.json rename to plugin/metaforge-plugin-db2/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-dm/src/main/resources/org/devlive/metaforge/plugin/DmService.capabilities.json b/plugin/metaforge-plugin-dm/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-dm/src/main/resources/org/devlive/metaforge/plugin/DmService.capabilities.json rename to plugin/metaforge-plugin-dm/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-dolphindb/src/main/resources/org/devlive/metaforge/plugin/dolphindb/DolphinDBService.capabilities.json b/plugin/metaforge-plugin-dolphindb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-dolphindb/src/main/resources/org/devlive/metaforge/plugin/dolphindb/DolphinDBService.capabilities.json rename to plugin/metaforge-plugin-dolphindb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-doris/src/main/resources/org/devlive/metaforge/plugin/DorisService.capabilities.json b/plugin/metaforge-plugin-doris/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-doris/src/main/resources/org/devlive/metaforge/plugin/DorisService.capabilities.json rename to plugin/metaforge-plugin-doris/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-dremio/src/main/resources/org/devlive/metaforge/plugin/DremioService.capabilities.json b/plugin/metaforge-plugin-dremio/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-dremio/src/main/resources/org/devlive/metaforge/plugin/DremioService.capabilities.json rename to plugin/metaforge-plugin-dremio/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-druid/src/main/resources/org/devlive/metaforge/plugin/DruidService.capabilities.json b/plugin/metaforge-plugin-druid/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-druid/src/main/resources/org/devlive/metaforge/plugin/DruidService.capabilities.json rename to plugin/metaforge-plugin-druid/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-duckdb/src/main/resources/org/devlive/metaforge/plugin/DuckDBService.capabilities.json b/plugin/metaforge-plugin-duckdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-duckdb/src/main/resources/org/devlive/metaforge/plugin/DuckDBService.capabilities.json rename to plugin/metaforge-plugin-duckdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-elasticsearch-8x/src/main/resources/org/devlive/metaforge/plugin/ElasticSearch8xService.capabilities.json b/plugin/metaforge-plugin-elasticsearch-8x/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-elasticsearch-8x/src/main/resources/org/devlive/metaforge/plugin/ElasticSearch8xService.capabilities.json rename to plugin/metaforge-plugin-elasticsearch-8x/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-greptimedb/src/main/resources/org/devlive/metaforge/plugin/GreptimeDBService.capabilities.json b/plugin/metaforge-plugin-greptimedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-greptimedb/src/main/resources/org/devlive/metaforge/plugin/GreptimeDBService.capabilities.json rename to plugin/metaforge-plugin-greptimedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-h2-tcp/src/main/resources/org/devlive/metaforge/plugin/H2TCPService.capabilities.json b/plugin/metaforge-plugin-h2-tcp/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-h2-tcp/src/main/resources/org/devlive/metaforge/plugin/H2TCPService.capabilities.json rename to plugin/metaforge-plugin-h2-tcp/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-h2/src/main/resources/org/devlive/metaforge/plugin/H2Service.capabilities.json b/plugin/metaforge-plugin-h2/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-h2/src/main/resources/org/devlive/metaforge/plugin/H2Service.capabilities.json rename to plugin/metaforge-plugin-h2/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-hdfs/src/main/resources/org/devlive/metaforge/plugin/HdfsService.capabilities.json b/plugin/metaforge-plugin-hdfs/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-hdfs/src/main/resources/org/devlive/metaforge/plugin/HdfsService.capabilities.json rename to plugin/metaforge-plugin-hdfs/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-hive-2x/src/main/resources/org/devlive/metaforge/plugin/Hive2xService.capabilities.json b/plugin/metaforge-plugin-hive-2x/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-hive-2x/src/main/resources/org/devlive/metaforge/plugin/Hive2xService.capabilities.json rename to plugin/metaforge-plugin-hive-2x/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-hologres/src/main/resources/org/devlive/metaforge/plugin/HologresService.capabilities.json b/plugin/metaforge-plugin-hologres/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-hologres/src/main/resources/org/devlive/metaforge/plugin/HologresService.capabilities.json rename to plugin/metaforge-plugin-hologres/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-ignite/src/main/resources/org/devlive/metaforge/plugin/IgniteService.capabilities.json b/plugin/metaforge-plugin-ignite/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-ignite/src/main/resources/org/devlive/metaforge/plugin/IgniteService.capabilities.json rename to plugin/metaforge-plugin-ignite/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-impala/src/main/resources/org/devlive/metaforge/plugin/ImpalaService.capabilities.json b/plugin/metaforge-plugin-impala/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-impala/src/main/resources/org/devlive/metaforge/plugin/ImpalaService.capabilities.json rename to plugin/metaforge-plugin-impala/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-influxdb/src/main/resources/org/devlive/metaforge/plugin/influxdb/InfluxdbService.capabilities.json b/plugin/metaforge-plugin-influxdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-influxdb/src/main/resources/org/devlive/metaforge/plugin/influxdb/InfluxdbService.capabilities.json rename to plugin/metaforge-plugin-influxdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-iotdb/src/main/resources/org/devlive/metaforge/plugin/IoTDBService.capabilities.json b/plugin/metaforge-plugin-iotdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-iotdb/src/main/resources/org/devlive/metaforge/plugin/IoTDBService.capabilities.json rename to plugin/metaforge-plugin-iotdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-kafka/src/main/resources/org/devlive/metaforge/plugin/KafkaService.capabilities.json b/plugin/metaforge-plugin-kafka/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-kafka/src/main/resources/org/devlive/metaforge/plugin/KafkaService.capabilities.json rename to plugin/metaforge-plugin-kafka/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-kylin/src/main/resources/org/devlive/metaforge/plugin/KylinService.capabilities.json b/plugin/metaforge-plugin-kylin/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-kylin/src/main/resources/org/devlive/metaforge/plugin/KylinService.capabilities.json rename to plugin/metaforge-plugin-kylin/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-kyuubi/src/main/resources/org/devlive/metaforge/plugin/KyuubiService.capabilities.json b/plugin/metaforge-plugin-kyuubi/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-kyuubi/src/main/resources/org/devlive/metaforge/plugin/KyuubiService.capabilities.json rename to plugin/metaforge-plugin-kyuubi/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-matrixone/src/main/resources/org/devlive/metaforge/plugin/MatrixOneService.capabilities.json b/plugin/metaforge-plugin-matrixone/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-matrixone/src/main/resources/org/devlive/metaforge/plugin/MatrixOneService.capabilities.json rename to plugin/metaforge-plugin-matrixone/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-monetdb/src/main/resources/org/devlive/metaforge/plugin/MonetDBService.capabilities.json b/plugin/metaforge-plugin-monetdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-monetdb/src/main/resources/org/devlive/metaforge/plugin/MonetDBService.capabilities.json rename to plugin/metaforge-plugin-monetdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-mongo-atlas/src/main/resources/org/devlive/metaforge/plugin/MongoAtlasService.capabilities.json b/plugin/metaforge-plugin-mongo-atlas/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-mongo-atlas/src/main/resources/org/devlive/metaforge/plugin/MongoAtlasService.capabilities.json rename to plugin/metaforge-plugin-mongo-atlas/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-mongo-community/src/main/resources/org/devlive/metaforge/plugin/MongoCommunityService.capabilities.json b/plugin/metaforge-plugin-mongo-community/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-mongo-community/src/main/resources/org/devlive/metaforge/plugin/MongoCommunityService.capabilities.json rename to plugin/metaforge-plugin-mongo-community/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-mysql/src/main/resources/org/devlive/metaforge/plugin/MySQLService.capabilities.json b/plugin/metaforge-plugin-mysql/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-mysql/src/main/resources/org/devlive/metaforge/plugin/MySQLService.capabilities.json rename to plugin/metaforge-plugin-mysql/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-neo4j/src/main/resources/org/devlive/metaforge/plugin/Neo4jService.capabilities.json b/plugin/metaforge-plugin-neo4j/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-neo4j/src/main/resources/org/devlive/metaforge/plugin/Neo4jService.capabilities.json rename to plugin/metaforge-plugin-neo4j/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-oceanbase/src/main/resources/org/devlive/metaforge/plugin/OceanBaseService.capabilities.json b/plugin/metaforge-plugin-oceanbase/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-oceanbase/src/main/resources/org/devlive/metaforge/plugin/OceanBaseService.capabilities.json rename to plugin/metaforge-plugin-oceanbase/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-oracle/src/main/resources/org/devlive/metaforge/plugin/OracleService.capabilities.json b/plugin/metaforge-plugin-oracle/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-oracle/src/main/resources/org/devlive/metaforge/plugin/OracleService.capabilities.json rename to plugin/metaforge-plugin-oracle/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-paradedb/src/main/resources/org/devlive/metaforge/plugin/ParadeDBService.capabilities.json b/plugin/metaforge-plugin-paradedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-paradedb/src/main/resources/org/devlive/metaforge/plugin/ParadeDBService.capabilities.json rename to plugin/metaforge-plugin-paradedb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-phoenix/src/main/resources/org/devlive/metaforge/plugin/PhoenixService.capabilities.json b/plugin/metaforge-plugin-phoenix/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-phoenix/src/main/resources/org/devlive/metaforge/plugin/PhoenixService.capabilities.json rename to plugin/metaforge-plugin-phoenix/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-pinot/src/main/resources/org/devlive/metaforge/plugin/PinotService.capabilities.json b/plugin/metaforge-plugin-pinot/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-pinot/src/main/resources/org/devlive/metaforge/plugin/PinotService.capabilities.json rename to plugin/metaforge-plugin-pinot/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-postgresql/src/main/resources/org/devlive/metaforge/plugin/PostgreSQLService.capabilities.json b/plugin/metaforge-plugin-postgresql/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-postgresql/src/main/resources/org/devlive/metaforge/plugin/PostgreSQLService.capabilities.json rename to plugin/metaforge-plugin-postgresql/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-presto/src/main/resources/org/devlive/metaforge/plugin/PrestoService.capabilities.json b/plugin/metaforge-plugin-presto/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-presto/src/main/resources/org/devlive/metaforge/plugin/PrestoService.capabilities.json rename to plugin/metaforge-plugin-presto/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-questdb/src/main/resources/org/devlive/metaforge/plugin/QuestDBService.capabilities.json b/plugin/metaforge-plugin-questdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-questdb/src/main/resources/org/devlive/metaforge/plugin/QuestDBService.capabilities.json rename to plugin/metaforge-plugin-questdb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-redis/src/main/resources/org/devlive/metaforge/plugin/RedisService.capabilities.json b/plugin/metaforge-plugin-redis/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-redis/src/main/resources/org/devlive/metaforge/plugin/RedisService.capabilities.json rename to plugin/metaforge-plugin-redis/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-scylladb/src/main/resources/org/devlive/metaforge/plugin/ScyllaDBService.capabilities.json b/plugin/metaforge-plugin-scylladb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-scylladb/src/main/resources/org/devlive/metaforge/plugin/ScyllaDBService.capabilities.json rename to plugin/metaforge-plugin-scylladb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-snowflake/src/main/resources/org/devlive/metaforge/plugin/SnowflakeService.capabilities.json b/plugin/metaforge-plugin-snowflake/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-snowflake/src/main/resources/org/devlive/metaforge/plugin/SnowflakeService.capabilities.json rename to plugin/metaforge-plugin-snowflake/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-solr/src/main/resources/org/devlive/metaforge/plugin/solr/SolrService.capabilities.json b/plugin/metaforge-plugin-solr/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-solr/src/main/resources/org/devlive/metaforge/plugin/solr/SolrService.capabilities.json rename to plugin/metaforge-plugin-solr/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-sqlserver/src/main/resources/org/devlive/metaforge/plugin/SqlServerService.capabilities.json b/plugin/metaforge-plugin-sqlserver/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-sqlserver/src/main/resources/org/devlive/metaforge/plugin/SqlServerService.capabilities.json rename to plugin/metaforge-plugin-sqlserver/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-starrocks/src/main/resources/org/devlive/metaforge/plugin/StarRocksService.capabilities.json b/plugin/metaforge-plugin-starrocks/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-starrocks/src/main/resources/org/devlive/metaforge/plugin/StarRocksService.capabilities.json rename to plugin/metaforge-plugin-starrocks/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-tdengine/src/main/resources/org/devlive/metaforge/plugin/TDengineService.capabilities.json b/plugin/metaforge-plugin-tdengine/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-tdengine/src/main/resources/org/devlive/metaforge/plugin/TDengineService.capabilities.json rename to plugin/metaforge-plugin-tdengine/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-timescale/src/main/resources/org/devlive/metaforge/plugin/TimescaleService.capabilities.json b/plugin/metaforge-plugin-timescale/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-timescale/src/main/resources/org/devlive/metaforge/plugin/TimescaleService.capabilities.json rename to plugin/metaforge-plugin-timescale/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-trino/src/main/resources/org/devlive/metaforge/plugin/TrinoService.capabilities.json b/plugin/metaforge-plugin-trino/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-trino/src/main/resources/org/devlive/metaforge/plugin/TrinoService.capabilities.json rename to plugin/metaforge-plugin-trino/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-ydb/src/main/resources/org/devlive/metaforge/plugin/YDBService.capabilities.json b/plugin/metaforge-plugin-ydb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-ydb/src/main/resources/org/devlive/metaforge/plugin/YDBService.capabilities.json rename to plugin/metaforge-plugin-ydb/src/main/resources/org/devlive/metaforge/plugin/capabilities.json diff --git a/plugin/metaforge-plugin-zookeeper/src/main/resources/org/devlive/metaforge/plugin/ZookeeperService.capabilities.json b/plugin/metaforge-plugin-zookeeper/src/main/resources/org/devlive/metaforge/plugin/capabilities.json similarity index 100% rename from plugin/metaforge-plugin-zookeeper/src/main/resources/org/devlive/metaforge/plugin/ZookeeperService.capabilities.json rename to plugin/metaforge-plugin-zookeeper/src/main/resources/org/devlive/metaforge/plugin/capabilities.json From 2d078332d14ad61a5d5aaf7b80da60170bb5ba98 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 21:55:59 -0400 Subject: [PATCH 003/232] feat(convert): let format plugins declare their writer settings ConvertService gains options() and extension(): a plugin lists the settings its writer takes, with bilingual labels and choices, and reads the values from ConvertRequest.options. - JSON: array or JSON Lines, indentation, date format (ISO 8601, yyyy-MM-dd HH:mm:ss or epoch milliseconds) and whether NULL fields are kept - CSV: delimiter, header row and NULL text; cells holding the delimiter, quotes or line breaks are now quoted - TXT: delimiter, header row and NULL text - XML: root and row element names and whether NULL attributes are kept --- .../convert/csv/CsvConvertService.kt | 40 ++++++- .../convert/json/JsonConvertService.kt | 88 ++++++++++++--- .../metaforge/convert/ConvertService.kt | 13 +++ .../metaforge/convert/model/ConvertOption.kt | 23 ++++ .../metaforge/convert/model/ConvertRequest.kt | 11 ++ .../metaforge/convert/TxtConvertService.kt | 25 ++++- .../convert/xml/XmlConvertService.kt | 20 +++- test/metaforge-test-convert/pom.xml | 18 ++++ .../test/options/WriterOptionsTest.java | 100 ++++++++++++++++++ test/metaforge-test-convert/test.xml | 1 + 10 files changed, 309 insertions(+), 30 deletions(-) create mode 100644 convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt create mode 100644 test/metaforge-test-convert/src/test/java/org/devlive/metaforge/test/options/WriterOptionsTest.java create mode 100644 test/metaforge-test-convert/test.xml diff --git a/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt b/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt index e6a1a19585..00df35c9f7 100644 --- a/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt +++ b/convert/metaforge-convert-csv/src/main/kotlin/org/devlive/metaforge/convert/csv/CsvConvertService.kt @@ -4,7 +4,9 @@ import com.google.common.base.Preconditions.checkState import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.apache.commons.csv.CSVFormat import org.apache.commons.io.FileUtils @@ -75,6 +77,18 @@ class CsvConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("delimiter", mapOf("zh" to "分隔符", "en" to "Delimiter"), ",", + listOf(choice(",", "逗号 ,", "Comma ,"), choice(";", "分号 ;", "Semicolon ;"), choice("\t", "制表符", "Tab"), choice("|", "竖线 |", "Pipe |"))), + ConvertOption("header", mapOf("zh" to "首行字段名", "en" to "Header row"), "true", + listOf(choice("true", "包含", "Included"), choice("false", "不包含", "Left out"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "empty", + listOf(choice("empty", "空单元格", "Empty cell"), choice("NULL", "写为 NULL", "Write NULL"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -83,23 +97,29 @@ class CsvConvertService : ConvertService log.info("${name()} writer start time [ ${DateUtils.now()} ]") response.headers = request.headers val file = FileConvert.formatFile(request, name()) - FileUtils.writeStringToFile(file, response.headers.joinToString(","), Charsets.UTF_8) + val delimiter = request.option("delimiter", ",").replace("\\t", "\t") + val nullText = if (request.excludeNull) "" else request.option("nulls", "empty").let { if (it == "NULL") "NULL" else "" } + val lines = mutableListOf() + if (request.option("header", "true") != "false") + { + lines.add(response.headers.joinToString(delimiter) { csvCell(it.toString(), delimiter) }) + } // The column types follow the header as a second line. if (request.includeStructure) { - FileUtils.writeStringToFile(file, "\n" + request.headers.indices.joinToString(",") { request.types.getOrNull(it)?.toString() ?: "" }, Charsets.UTF_8, true) + lines.add(request.headers.indices.joinToString(delimiter) { csvCell(request.types.getOrNull(it)?.toString() ?: "", delimiter) }) } val columns = mutableListOf() request.columns .forEach { - FileUtils.writeStringToFile(file, "\n", Charsets.UTF_8, true) when (it) { - is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = ",") { value -> if (value == null && request.excludeNull) "" else value.toString() }, Charsets.UTF_8, true) - else -> FileUtils.writeStringToFile(file, it.toString(), Charsets.UTF_8, true) + is List<*> -> lines.add(it.joinToString(separator = delimiter) { value -> if (value == null) nullText else csvCell(value.toString(), delimiter) }) + else -> lines.add(it.toString()) } } + FileUtils.writeStringToFile(file, lines.joinToString("\n"), Charsets.UTF_8) response.columns = columns log.info("${name()} writer end time [ ${DateUtils.now()} ]") response.successful = true @@ -145,3 +165,13 @@ class CsvConvertService : ConvertService return response } } + +/** A cell quoted when it holds the delimiter, a quote or a line break, quotes doubled (RFC 4180). */ +internal fun csvCell(value: String, delimiter: String): String +{ + return if (value.contains(delimiter) || value.contains('"') || value.contains('\n') || value.contains('\r')) + { + "\"" + value.replace("\"", "\"\"") + "\"" + } + else value +} diff --git a/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt b/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt index 6868f709e4..e8b0515236 100644 --- a/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt +++ b/convert/metaforge-convert-json/src/main/kotlin/org/devlive/metaforge/convert/json/JsonConvertService.kt @@ -2,6 +2,8 @@ package org.devlive.metaforge.convert.json import com.fasterxml.jackson.core.JsonEncoding import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.util.DefaultIndenter +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode @@ -9,13 +11,16 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert.formatFile +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.slf4j.LoggerFactory.getLogger import java.io.BufferedReader import java.io.IOException import java.io.InputStreamReader import java.time.LocalDateTime +import java.time.ZoneId import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException import java.util.Objects.requireNonNull @@ -131,6 +136,20 @@ class JsonConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("root", mapOf("zh" to "JSON 根结构", "en" to "JSON root"), "array", + listOf(choice("array", "数组 Array", "Array"), choice("lines", "每行一个对象 JSON Lines", "One object per line (JSON Lines)"))), + ConvertOption("indent", mapOf("zh" to "缩进格式", "en" to "Indentation"), "2", + listOf(choice("0", "不缩进", "None"), choice("2", "2 个空格", "2 spaces"), choice("4", "4 个空格", "4 spaces"))), + ConvertOption("dateFormat", mapOf("zh" to "日期输出格式", "en" to "Date format"), "iso", + listOf(choice("iso", "ISO 8601", "ISO 8601"), choice("datetime", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss"), choice("epoch", "时间戳（毫秒）", "Epoch milliseconds"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "keep", + listOf(choice("keep", "保留 null", "Keep null"), choice("omit", "省略字段", "Leave the field out"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -141,9 +160,20 @@ class JsonConvertService : ConvertService val file = formatFile(request, name()) log.info("${name()} writer file absolute path [ ${file.absolutePath} ]") + val lines = request.option("root", "array") == "lines" + val indent = request.option("indent", "2").toIntOrNull() ?: 2 + val dateFormat = request.option("dateFormat", "iso") + val omitNull = request.excludeNull || request.option("nulls", "keep") == "omit" val factory = JsonFactory() + factory.setRootValueSeparator(if (lines) "\n" else " ") factory.createGenerator(file, JsonEncoding.UTF8).use { generator -> generator.codec = ObjectMapper() + // JSON Lines keeps every record on its own line, so it is never indented. + if (indent > 0 && ! lines) + { + val indenter = DefaultIndenter(" ".repeat(indent), DefaultIndenter.SYS_LF) + generator.prettyPrinter = DefaultPrettyPrinter().withObjectIndenter(indenter).withArrayIndenter(indenter) + } if (request.includeStructure) { generator.writeStartObject() @@ -155,9 +185,19 @@ class JsonConvertService : ConvertService generator.writeEndObject() } generator.writeEndArray() - generator.writeFieldName("rows") + if (lines) + { + generator.writeEndObject() + } + else + { + generator.writeFieldName("rows") + } + } + if (! lines) + { + generator.writeStartArray() } - generator.writeStartArray() request.columns.forEach { column -> generator.writeStartObject() for (headerIndex in request.headers.indices) @@ -167,25 +207,18 @@ class JsonConvertService : ConvertService { is List<*> -> { - val value = column[headerIndex] - if (value == null && request.excludeNull) + val value = dateValue(column[headerIndex], dateFormat) + if (value == null && omitNull) { continue } - if (value is LocalDateTime) - { - generator.writeStringField(header, value.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)) - } - else - { - generator.writeObjectField(header, value) - } + generator.writeObjectField(header, value) } is ObjectNode -> { val value = column.get(header) - if ((value == null || value.isNull) && request.excludeNull) + if ((value == null || value.isNull) && omitNull) { continue } @@ -212,10 +245,13 @@ class JsonConvertService : ConvertService } generator.writeEndObject() } - generator.writeEndArray() - if (request.includeStructure) + if (! lines) { - generator.writeEndObject() + generator.writeEndArray() + if (request.includeStructure) + { + generator.writeEndObject() + } } } @@ -232,7 +268,25 @@ class JsonConvertService : ConvertService return response } - private fun isDateTime(text: String): Boolean + /** A date or time value in the chosen format: ISO 8601, yyyy-MM-dd HH:mm:ss or epoch milliseconds. */ +private fun dateValue(value: Any?, format: String): Any? +{ + val time = when (value) + { + is LocalDateTime -> value + is java.sql.Timestamp -> value.toLocalDateTime() + is java.util.Date -> LocalDateTime.ofInstant(value.toInstant(), ZoneId.systemDefault()) + else -> return value + } + return when (format) + { + "epoch" -> time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() + "datetime" -> time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + else -> time.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + } +} + +private fun isDateTime(text: String): Boolean { return try { diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt index 70890a4e90..0d7b2fa512 100644 --- a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/ConvertService.kt @@ -1,5 +1,6 @@ package org.devlive.metaforge.convert +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest import org.devlive.metaforge.convert.model.ConvertResponse import org.devlive.metaforge.plugin.Service @@ -18,6 +19,18 @@ interface ConvertService : Service return "Integrate ${name()} format convert" } + /** Writer settings the plugin offers; values arrive in [ConvertRequest.options]. */ + fun options(): List + { + return listOf() + } + + /** Extension of the files the writer produces. */ + fun extension(): String + { + return name().lowercase() + } + fun format(request: ConvertRequest): ConvertResponse fun formatStream(request: ConvertRequest): ConvertResponse diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt new file mode 100644 index 0000000000..a682054d54 --- /dev/null +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertOption.kt @@ -0,0 +1,23 @@ +package org.devlive.metaforge.convert.model + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings + +/** + * A setting a convert plugin declares for its writer, shown by the export page as a field: a + * choice among [choices] or free text when there are none. Labels are keyed by language (zh, en). + */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "Read-only description of a plugin setting") +data class ConvertOption( + val name: String, + val labels: Map, + val defaultValue: String, + val choices: List = listOf() +) + +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "Read-only description of a choice") +data class ConvertChoice( + val value: String, + val labels: Map +) + +fun choice(value: String, zh: String, en: String) = ConvertChoice(value, mapOf("zh" to zh, "en" to en)) diff --git a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt index 93749726c9..b3f1fcdb25 100644 --- a/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt +++ b/convert/metaforge-convert-spi/src/main/kotlin/org/devlive/metaforge/convert/model/ConvertRequest.kt @@ -1,7 +1,9 @@ package org.devlive.metaforge.convert.model +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import java.io.InputStream +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"], justification = "A request carries the caller's values") class ConvertRequest { private var _headers: List = listOf() @@ -15,6 +17,15 @@ class ConvertRequest // Currently only valid for txt format var delimiter: String? = null + // Values of the plugin's declared options (see ConvertService.options), by option name + var options: Map = mapOf() + + /** An option's value, or [fallback] when it was not given. */ + fun option(name: String, fallback: String): String + { + return options[name]?.takeIf { it.isNotEmpty() } ?: fallback + } + // The format type of the original data var format: String? = null diff --git a/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt b/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt index 54f7c2c49e..c5cbf7f855 100644 --- a/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt +++ b/convert/metaforge-convert-txt/src/main/kotlin/org/devlive/metaforge/convert/TxtConvertService.kt @@ -3,7 +3,9 @@ package org.devlive.metaforge.convert import com.google.common.base.Preconditions.checkArgument import com.google.common.base.Preconditions.checkState import org.devlive.metaforge.common.utils.DateUtils +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.apache.commons.io.FileUtils import org.apache.commons.lang3.ObjectUtils.isNotEmpty @@ -96,12 +98,28 @@ class TxtConvertService : ConvertService return response } + override fun options(): List + { + return listOf( + ConvertOption("delimiter", mapOf("zh" to "分隔符", "en" to "Delimiter"), "\t", + listOf(choice("\t", "制表符", "Tab"), choice(",", "逗号 ,", "Comma ,"), choice(" ", "空格", "Space"), choice("|", "竖线 |", "Pipe |"))), + ConvertOption("header", mapOf("zh" to "首行字段名", "en" to "Header row"), "true", + listOf(choice("true", "包含", "Included"), choice("false", "不包含", "Left out"))), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "empty", + listOf(choice("empty", "留空", "Empty"), choice("NULL", "写为 NULL", "Write NULL"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() try { + // A declared delimiter option wins over the request's delimiter. + request.options["delimiter"]?.takeIf { it.isNotEmpty() }?.let { request.delimiter = it.replace("\\t", "\t") } checkArgument(isNotEmpty(request.delimiter), "Delimiter must not be empty") + val header = request.option("header", "true") != "false" + val nullText = if (request.excludeNull) "" else request.option("nulls", "empty").let { if (it == "NULL") "NULL" else "" } log.info("${name()} writer origin path [ ${request.path} ]") log.info("${name()} writer start time [ ${DateUtils.now()} ]") @@ -111,10 +129,7 @@ class TxtConvertService : ConvertService request.delimiter ?.let { delimiter -> log.info("${name()} writer file headers start") - request.headers - .let { headers -> - FileUtils.writeStringToFile(file, headers.joinToString(separator = delimiter), Charsets.UTF_8) - } + FileUtils.writeStringToFile(file, if (header) request.headers.joinToString(separator = delimiter) else "", Charsets.UTF_8) // The column types follow the header as a second line. if (request.includeStructure) { @@ -129,7 +144,7 @@ class TxtConvertService : ConvertService FileUtils.writeStringToFile(file, "\n", Charsets.UTF_8, true) when (it) { - is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = delimiter) { value -> if (value == null && request.excludeNull) "" else value.toString() }, + is List<*> -> FileUtils.writeStringToFile(file, it.joinToString(separator = delimiter) { value -> value?.toString() ?: nullText }, Charsets.UTF_8, true) else -> FileUtils.writeStringToFile(file, it.toString(), Charsets.UTF_8, true) } diff --git a/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt b/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt index 10e180da54..b37c9faa99 100644 --- a/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt +++ b/convert/metaforge-convert-xml/src/main/kotlin/org/devlive/metaforge/convert/xml/XmlConvertService.kt @@ -6,7 +6,9 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper import org.devlive.metaforge.common.utils.DateUtils import org.devlive.metaforge.convert.ConvertService import org.devlive.metaforge.convert.FileConvert.formatFile +import org.devlive.metaforge.convert.model.ConvertOption import org.devlive.metaforge.convert.model.ConvertRequest +import org.devlive.metaforge.convert.model.choice import org.devlive.metaforge.convert.model.ConvertResponse import org.slf4j.LoggerFactory.getLogger import java.io.ByteArrayOutputStream @@ -66,6 +68,16 @@ class XmlConvertService : ConvertService TODO("Not yet implemented") } + override fun options(): List + { + return listOf( + ConvertOption("rootElement", mapOf("zh" to "根元素名", "en" to "Root element"), root), + ConvertOption("rowElement", mapOf("zh" to "行元素名", "en" to "Row element"), node), + ConvertOption("nulls", mapOf("zh" to "NULL 值处理", "en" to "NULL values"), "keep", + listOf(choice("keep", "写为 null", "Write null"), choice("omit", "省略属性", "Leave the attribute out"))) + ) + } + override fun writer(request: ConvertRequest): ConvertResponse { val response = ConvertResponse() @@ -80,7 +92,9 @@ class XmlConvertService : ConvertService .use { generator -> generator.codec = XmlMapper() val staxWriter = generator.staxWriter - staxWriter.writeStartElement(root) + val rowElement = request.option("rowElement", node) + val omitNull = request.excludeNull || request.option("nulls", "keep") == "omit" + staxWriter.writeStartElement(request.option("rootElement", root)) if (request.includeStructure) { staxWriter.writeStartElement("Columns") @@ -94,7 +108,7 @@ class XmlConvertService : ConvertService } request.columns .forEach { column -> - staxWriter.writeStartElement(node) + staxWriter.writeStartElement(rowElement) for (headerIndex in request.headers.indices) { when (column) @@ -102,7 +116,7 @@ class XmlConvertService : ConvertService is List<*> -> { val value = column[headerIndex] - if (value != null || !request.excludeNull) + if (value != null || ! omitNull) { staxWriter.writeAttribute(request.headers[headerIndex].toString(), value.toString()) } diff --git a/test/metaforge-test-convert/pom.xml b/test/metaforge-test-convert/pom.xml index 827a48e9de..97dacc3160 100644 --- a/test/metaforge-test-convert/pom.xml +++ b/test/metaforge-test-convert/pom.xml @@ -43,6 +43,24 @@ ${project.version} test + + org.devlive.metaforge + metaforge-convert-json + ${project.version} + test + + + org.devlive.metaforge + metaforge-convert-csv + ${project.version} + test + + + org.devlive.metaforge + metaforge-convert-txt + ${project.version} + test + diff --git a/test/metaforge-test-convert/src/test/java/org/devlive/metaforge/test/options/WriterOptionsTest.java b/test/metaforge-test-convert/src/test/java/org/devlive/metaforge/test/options/WriterOptionsTest.java new file mode 100644 index 0000000000..a60ae7f72e --- /dev/null +++ b/test/metaforge-test-convert/src/test/java/org/devlive/metaforge/test/options/WriterOptionsTest.java @@ -0,0 +1,100 @@ +package org.devlive.metaforge.test.options; + +import org.devlive.metaforge.convert.ConvertService; +import org.devlive.metaforge.convert.TxtConvertService; +import org.devlive.metaforge.convert.csv.CsvConvertService; +import org.devlive.metaforge.convert.json.JsonConvertService; +import org.devlive.metaforge.convert.model.ConvertOption; +import org.devlive.metaforge.convert.model.ConvertRequest; +import org.devlive.metaforge.convert.model.ConvertResponse; +import org.devlive.metaforge.convert.xml.XmlConvertService; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** Every writer setting a convert plugin declares changes the file it writes. */ +public class WriterOptionsTest +{ + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + private String write(ConvertService service, Map options) + throws IOException + { + File directory = folder.newFolder(); + ConvertRequest request = new ConvertRequest(); + request.setName("t"); + request.setPath(directory.getAbsolutePath()); + request.setDelimiter("\t"); + request.setHeaders(List.of("id", "name", "at")); + request.setColumns(List.of(Arrays.asList(1, "a,b", Timestamp.valueOf("2026-10-30 09:20:00")), Arrays.asList(2, null, null))); + request.setOptions(options); + ConvertResponse response = service.writer(request); + Assert.assertTrue(response.getMessage(), Boolean.TRUE.equals(response.getSuccessful())); + File[] files = directory.listFiles(); + Assert.assertNotNull(files); + Assert.assertEquals(1, files.length); + Assert.assertTrue(files[0].getName(), files[0].getName().endsWith("." + service.extension())); + return Files.readString(files[0].toPath(), StandardCharsets.UTF_8); + } + + private static List names(ConvertService service) + { + return service.options().stream().map(ConvertOption::getName).collect(Collectors.toList()); + } + + @Test + public void jsonWritesArraysOrLinesWithTheChosenIndentDatesAndNulls() + throws IOException + { + JsonConvertService json = new JsonConvertService(); + Assert.assertEquals(List.of("root", "indent", "dateFormat", "nulls"), names(json)); + String pretty = write(json, Map.of("indent", "2", "dateFormat", "datetime")); + Assert.assertTrue(pretty, pretty.startsWith("[ {\n \"id\" : 1,") || pretty.startsWith("[\n {")); + Assert.assertTrue(pretty, pretty.contains("\"2026-10-30 09:20:00\"")); + Assert.assertTrue(pretty, pretty.contains("\"name\" : null")); + String lines = write(json, Map.of("root", "lines", "nulls", "omit", "dateFormat", "epoch")); + String[] rows = lines.trim().split("\n"); + Assert.assertEquals(lines, 2, rows.length); + Assert.assertTrue(rows[0], rows[0].startsWith("{\"id\":1,\"name\":\"a,b\",\"at\":") && !rows[0].contains("2026")); + Assert.assertEquals("{\"id\":2}", rows[1]); + } + + @Test + public void csvQuotesCellsAndHonoursDelimiterHeaderAndNulls() + throws IOException + { + CsvConvertService csv = new CsvConvertService(); + Assert.assertEquals(List.of("delimiter", "header", "nulls"), names(csv)); + Assert.assertEquals("id,name,at\n1,\"a,b\",2026-10-30 09:20:00.0\n2,,", write(csv, Map.of())); + Assert.assertEquals("1;a,b;2026-10-30 09:20:00.0\n2;NULL;NULL", write(csv, Map.of("delimiter", ";", "header", "false", "nulls", "NULL"))); + } + + @Test + public void txtTakesItsDelimiterAndHeaderFromTheOptions() + throws IOException + { + TxtConvertService txt = new TxtConvertService(); + Assert.assertEquals("1|a,b|2026-10-30 09:20:00.0\n2||", write(txt, Map.of("delimiter", "|", "header", "false")).replaceFirst("^\n", "")); + } + + @Test + public void xmlNamesItsElementsAndCanLeaveNullsOut() + throws IOException + { + String xml = write(new XmlConvertService(), Map.of("rootElement", "rows", "rowElement", "row", "nulls", "omit")); + Assert.assertTrue(xml, xml.startsWith("") || xml.contains("")); + } +} diff --git a/test/metaforge-test-convert/test.xml b/test/metaforge-test-convert/test.xml new file mode 100644 index 0000000000..afeef37ee0 --- /dev/null +++ b/test/metaforge-test-convert/test.xml @@ -0,0 +1 @@ + \ No newline at end of file From 9c3ca3b4cff01592f5d0eeb4324e4ba2d10a81c5 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:19:32 -0400 Subject: [PATCH 004/232] feat(export): rebuild data export as a five-step wizard Export now follows the prototype in the workbench: choose objects, set content per table, pick a format plugin with its declared settings and a live sample, set the output and execution, then check the plan and run it as a job whose file is downloaded when it is done. The page fits the viewport and only the step content scrolls. --- .../src/composables/useExportTool.ts | 63 ----- .../src/composables/useExportWizard.ts | 228 ++++++++++++++++++ .../src/composables/useSourceCascade.ts | 65 ----- .../src/composables/useSourceDatabase.ts | 4 +- .../src/composables/useWorkbenchHeader.ts | 7 +- core/metaforge-web/src/i18n/locales/en.json | 212 +++++++++++++++- .../metaforge-web/src/i18n/locales/zh-cn.json | 212 +++++++++++++++- .../src/model/database-export.ts | 100 ++++++++ core/metaforge-web/src/model/source-manage.ts | 11 +- core/metaforge-web/src/services/export.ts | 40 +++ .../src/views/pages/tools/ExportHome.vue | 104 +++++--- .../views/pages/tools/export/ExportChoice.vue | 20 ++ .../pages/tools/export/ExportStepContent.vue | 98 ++++++++ .../pages/tools/export/ExportStepFormat.vue | 100 ++++++++ .../pages/tools/export/ExportStepObjects.vue | 94 ++++++++ .../pages/tools/export/ExportStepOutput.vue | 70 ++++++ .../pages/tools/export/ExportStepPreview.vue | 128 ++++++++++ .../tests/database-export.test.mjs | 52 ++++ 18 files changed, 1422 insertions(+), 186 deletions(-) delete mode 100644 core/metaforge-web/src/composables/useExportTool.ts create mode 100644 core/metaforge-web/src/composables/useExportWizard.ts delete mode 100644 core/metaforge-web/src/composables/useSourceCascade.ts create mode 100644 core/metaforge-web/src/model/database-export.ts create mode 100644 core/metaforge-web/src/services/export.ts create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportChoice.vue create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportStepContent.vue create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportStepFormat.vue create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportStepObjects.vue create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportStepOutput.vue create mode 100644 core/metaforge-web/src/views/pages/tools/export/ExportStepPreview.vue create mode 100644 core/metaforge-web/tests/database-export.test.mjs diff --git a/core/metaforge-web/src/composables/useExportTool.ts b/core/metaforge-web/src/composables/useExportTool.ts deleted file mode 100644 index 099d18d509..0000000000 --- a/core/metaforge-web/src/composables/useExportTool.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { computed, nextTick, onMounted, ref } from 'vue' -import { useRoute } from 'vue-router' -import { useI18n } from 'vue-i18n' -import { MfMessage } from 'metaforge-ui' -import MetadataService from '@/services/metadata' -import PluginService from '@/services/plugin' -import { emptyExportForm, exportBody, exportFormats } from '@/model/source-export' -import { useSourceCascade } from './useSourceCascade' - -/** - * Data export tool page: pick a source, database and table, an installed format and optionally a - * row count, and download the file from the same export the table dialog uses. A source and - * database passed in the address (from the database overview) are selected on arrival. - */ -export function useExportTool() { - const { t } = useI18n() - const route = useRoute() - const { sourceCode, databaseName, tableName, sourceOptions, databaseOptions, tableOptions } = useSourceCascade() - const format = ref('') - const formatOptions = ref<{ label: string; value: string }[]>([]) - // Empty exports every row; a number exports that many from the first row. - const count = ref('') - const exporting = ref(false) - const canExport = computed(() => !!(sourceCode.value && databaseName.value && tableName.value && format.value) && !exporting.value) - - async function onExport() { - if (!canExport.value) return - const limit = Number(count.value) - const body = exportBody({ ...emptyExportForm(), format: format.value, includeStructure: false, scope: limit >= 1 ? 'range' : 'all', count: limit >= 1 ? limit : 1 }) - exporting.value = true - try { - const response = await MetadataService.exportFile(sourceCode.value, databaseName.value, tableName.value, body) - if (!response.status || !response.blob) { MfMessage.error(response.message || t('common.loadingFailed')); return } - const url = URL.createObjectURL(response.blob) - const link = document.createElement('a') - link.href = url - link.download = response.fileName || tableName.value - link.click() - URL.revokeObjectURL(url) - MfMessage.success(t('source.export.downloaded', { file: link.download })) - } finally { - exporting.value = false - } - } - - onMounted(async () => { - const response = await PluginService.getPlugins() - if (response.status) { - formatOptions.value = exportFormats(response.data || []).map(item => ({ label: item.label, value: item.name })) - format.value = formatOptions.value[0]?.value || '' - } - const source = typeof route.query.source === 'string' ? route.query.source : '' - const database = typeof route.query.database === 'string' ? route.query.database : '' - if (source) { - sourceCode.value = source - // Changing the source clears the database first; pick it once that has run. - await nextTick() - if (database) databaseName.value = database - } - }) - - return { sourceCode, databaseName, tableName, sourceOptions, databaseOptions, tableOptions, format, formatOptions, count, exporting, canExport, onExport } -} diff --git a/core/metaforge-web/src/composables/useExportWizard.ts b/core/metaforge-web/src/composables/useExportWizard.ts new file mode 100644 index 0000000000..bc81d39a17 --- /dev/null +++ b/core/metaforge-web/src/composables/useExportWizard.ts @@ -0,0 +1,228 @@ +import { computed, onScopeDispose, ref, watch } from 'vue' +import { useRoute, useRouter } from 'vue-router' +import { useI18n } from 'vue-i18n' +import { MfMessage } from 'metaforge-ui' +import ExportService from '@/services/export' +import MetadataService from '@/services/metadata' +import { sourceManageContext, sourceManageLocation } from '@/model/source-manage' +import { useSourceCapabilities } from './useSourceCapabilities' +import { configured, defaultExportState, EXPORT_STEPS, exportedTables, exportPlan, formatOptions, outputName, setting, + type ExportColumn, type ExportFilter, type ExportFormat, type ExportObject, type TableSetting } from '@/model/database-export' + +export { EXPORT_STEPS } + +/** + * Five-step export of the database the workbench header shows: choose tables, set their content, + * fields and rows, pick an installed format and its declared settings, set the output, then check + * the plan on the server and run it as a job whose file is downloaded when it is done. + */ +export function useExportWizard() { + const { t } = useI18n() + const route = useRoute() + const router = useRouter() + const context = computed(() => sourceManageContext(route)) + const { can } = useSourceCapabilities(computed(() => context.value.source)) + // Structure goes into schema.sql only for sources whose plugin can give a table's statement. + const canStructure = computed(() => can('table:statement')) + + const step = ref(0) + const state = ref(defaultExportState('')) + const objects = ref([]) + const objectsLoading = ref(false) + const columns = ref>({}) + const formats = ref([]) + const current = ref('') + const sample = ref<{ file: string; content: string } | null>(null) + const sampleLoading = ref(false) + const sampleError = ref('') + const check = ref(null) + const checkLoading = ref(false) + const checkError = ref('') + const rows = ref<{ headers: string[]; rows: any[][] } | null>(null) + const job = ref(null) + const starting = ref(false) + let alive = true + let sampleTimer: ReturnType | undefined + let sampleRevision = 0 + let pollTimer: ReturnType | undefined + + const location = () => [context.value.source, context.value.database] as const + const tables = computed(() => exportedTables(state.value, objects.value)) + const format = computed(() => formats.value.find(item => item.name === state.value.format)) + const output = computed(() => outputName(state.value, tables.value, format.value?.extension || '')) + const plan = computed(() => exportPlan(state.value, objects.value, columns.value)) + const running = computed(() => job.value?.status === 'running') + const canNext = computed(() => { + if (step.value === 0) return tables.value.length > 0 + if (step.value === 2) return !!format.value + if (step.value === 3) return !!state.value.fileName.trim() && state.value.batchSize > 0 + return true + }) + const canStart = computed(() => !!check.value && check.value.errors === 0 && !checkLoading.value && !starting.value && !job.value) + + async function load() { + const [source, database] = location() + state.value = defaultExportState(database) + objects.value = [] + columns.value = {} + if (!source || !database) return + objectsLoading.value = true + const [sizes, list, formatResponse] = await Promise.all([ + MetadataService.getDatabaseTables(source, database), MetadataService.getTablesByDatabase(source, database), ExportService.formats(), + ]) + if (!alive) return + objectsLoading.value = false + const bySize = new Map((sizes.status ? sizes.data?.columns || [] : []).map((row: any) => [String(row.object_name), row])) + objects.value = (list.status ? list.data?.columns || [] : []) + .filter((row: any) => row.type_name === 'table' || row.type_name === 'view') + .map((row: any) => { + const size = bySize.get(String(row.object_name)) + const number = (value: any) => value === null || value === undefined || value === '' ? null : Number(value) + return { name: String(row.object_name), type: row.type_name, rows: number(size?.object_rows), bytes: number(size?.object_data_length), comment: String(row.object_comment || '') } + }) + formats.value = formatResponse.status ? formatResponse.data || [] : [] + if (!canStructure.value) state.value.content = 'data' + chooseFormat(formats.value[0]?.name || '') + } + + async function columnsOf(table: string): Promise { + if (columns.value[table]) return columns.value[table] + const [source, database] = location() + const response = await MetadataService.getColumnsByTable(source, database, table) + const list = response.status ? (response.data?.columns || []).filter((row: any) => row.type_name === 'column') + .map((row: any) => ({ name: String(row.object_name), type: String(row.object_data_type || '').toUpperCase() })) : [] + columns.value = { ...columns.value, [table]: list } + return list + } + + // ---- step 1 ---- + const toggle = (name: string, checked: boolean) => { + const selected = new Set(state.value.selected) + if (checked) selected.add(name); else selected.delete(name) + state.value.selected = objects.value.map(object => object.name).filter(item => selected.has(item)) + } + const selectAll = (names: string[]) => { state.value.selected = [...new Set([...state.value.selected, ...names])] } + const clearSelection = () => { state.value.selected = [] } + + // ---- step 2 ---- + async function configure(table: string) { + current.value = table + const list = await columnsOf(table) + if (!state.value.settings[table]) state.value.settings[table] = { fields: list.map(column => column.name), filters: [], scope: 'all' } + } + const currentSetting = computed(() => setting(state.value, current.value)) + const toggleField = (name: string, checked: boolean) => { + const value = state.value.settings[current.value] + if (!value) return + const order = (columns.value[current.value] || []).map(column => column.name) + const fields = new Set(value.fields) + if (checked) fields.add(name); else fields.delete(name) + value.fields = order.filter(item => fields.has(item)) + } + const addFilter = () => { + const value = state.value.settings[current.value] + if (!value) return + value.scope = 'filtered' + value.filters.push({ column: columns.value[current.value]?.[0]?.name || '', operator: '=', value: '' }) + } + const removeFilter = (filter: ExportFilter) => { + const value = state.value.settings[current.value] + if (value) value.filters = value.filters.filter(item => item !== filter) + } + const isConfigured = (table: string) => configured(state.value.settings[table]) + + // ---- step 3 ---- + function chooseFormat(name: string) { + state.value.format = name + state.value.options = formatOptions(formats.value.find(item => item.name === name), state.value.options) + } + // The format's output for the first table is written by the server as the settings change. + function scheduleSample() { + clearTimeout(sampleTimer) + sampleTimer = setTimeout(runSample, 300) + } + async function runSample() { + const table = tables.value[0] + const [source, database] = location() + if (!table || !state.value.format) { sample.value = null; return } + const revision = ++sampleRevision + sampleLoading.value = true + const response = await ExportService.sample(source, database, plan.value, table, 3) + if (revision !== sampleRevision || !alive) return + sampleLoading.value = false + sampleError.value = response.status ? '' : response.message || t('common.requestError') + sample.value = response.status ? { file: String(response.data?.file || ''), content: String(response.data?.content || '') } : null + } + watch(() => [step.value, state.value.format, JSON.stringify(state.value.options), tables.value[0]], () => { if (step.value === 2) scheduleSample() }) + + // ---- step 5 ---- + async function runCheck() { + const [source, database] = location() + checkLoading.value = true + checkError.value = '' + const [checked, sampled] = await Promise.all([ + ExportService.check(source, database, plan.value), + tables.value[0] ? ExportService.sample(source, database, { ...plan.value, format: '' }, tables.value[0], 3) : Promise.resolve(null), + ]) + if (!alive) return + checkLoading.value = false + checkError.value = checked.status ? '' : checked.message || t('common.requestError') + check.value = checked.status ? checked.data : null + rows.value = sampled?.status ? { headers: sampled.data?.headers || [], rows: sampled.data?.rows || [] } : null + } + + async function next() { + if (!canNext.value || step.value >= EXPORT_STEPS.length - 1) return + step.value++ + if (step.value === 1 && !current.value && tables.value[0]) await configure(tables.value[0]) + if (step.value === 4) runCheck() + } + const previous = () => { if (step.value > 0 && !running.value && !job.value) step.value-- } + const goTo = (index: number) => { if (index < step.value && !running.value && !job.value) step.value = index } + const back = () => router.push(sourceManageLocation({ source: context.value.source, database: context.value.database })) + + async function start() { + if (!canStart.value) return + const [source, database] = location() + starting.value = true + const response = await ExportService.execute(source, database, plan.value) + starting.value = false + if (!response.status) { MfMessage.error(response.message || t('common.requestError')); return } + job.value = { status: 'running', tables: [] } + poll(String(response.data?.jobId)) + } + async function poll(id: string) { + const response = await ExportService.job(id) + if (!alive) return + if (response.status) job.value = { ...response.data, id } + if (job.value?.status === 'running') { pollTimer = setTimeout(() => poll(id), 1000); return } + if (job.value?.status === 'done' || job.value?.status === 'partial') { + MfMessage.success(t('export.done')) + if (state.value.autoDownload) download() + } + else MfMessage.error(job.value?.message || t('export.failed')) + } + async function download() { + if (!job.value?.id) return + const response = await ExportService.file(job.value.id) + if (!response.status || !response.blob) { MfMessage.error(response.message || t('common.requestError')); return } + const url = URL.createObjectURL(response.blob) + const link = document.createElement('a') + link.href = url + link.download = response.fileName || output.value + link.click() + URL.revokeObjectURL(url) + } + + watch(() => [context.value.source, context.value.database], () => { step.value = 0; job.value = null; check.value = null; current.value = ''; sample.value = null; load() }, { immediate: true }) + onScopeDispose(() => { alive = false; sampleRevision++; clearTimeout(sampleTimer); clearTimeout(pollTimer) }) + + return { + context, can, canStructure, step, state, objects, objectsLoading, columns, formats, format, tables, output, current, currentSetting, sample, sampleLoading, + sampleError, check, checkLoading, checkError, rows, job, starting, running, canNext, canStart, + toggle, selectAll, clearSelection, configure, toggleField, addFilter, removeFilter, isConfigured, chooseFormat, runSample, runCheck, + next, previous, goTo, back, start, download, + } +} + +export type ExportWizard = ReturnType diff --git a/core/metaforge-web/src/composables/useSourceCascade.ts b/core/metaforge-web/src/composables/useSourceCascade.ts deleted file mode 100644 index 831c498909..0000000000 --- a/core/metaforge-web/src/composables/useSourceCascade.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { onMounted, ref, watch } from 'vue' -import { FilterModel } from '@/model/filter' -import SourceService from '@/services/source' -import MetadataService from '@/services/metadata' - -/** - * 数据源 → 数据库 → 数据表 三级级联选择（Vue3 Composable）。 - * - * 导入/导出/对比等工具页共用：切换数据源自动重载库，切换库自动重载表； - * 库/表随之清空。表列表仅保留基础表（type_name === 'table'）。 - */ -export function useSourceCascade() -{ - const sourceCode = ref('') - const databaseName = ref('') - const tableName = ref('') - const sourceOptions = ref<{ label: string; value: string }[]>([]) - const databaseOptions = ref<{ label: string; value: string }[]>([]) - const tableOptions = ref<{ label: string; value: string }[]>([]) - - const loadDatabases = () => { - databaseName.value = '' - tableName.value = '' - databaseOptions.value = [] - tableOptions.value = [] - if (!sourceCode.value) return - MetadataService.getDatabaseBySource(sourceCode.value) - .then(response => { - if (response.status) { - databaseOptions.value = (response.data?.columns || []) - .map((item: any) => ({ label: item.object_name || item.schema_name, value: item.object_name || item.schema_name })) - .filter((item: any) => item.value) - } - }) - } - - const loadTables = () => { - tableName.value = '' - tableOptions.value = [] - if (!sourceCode.value || !databaseName.value) return - MetadataService.getTablesByDatabase(sourceCode.value, databaseName.value) - .then(response => { - if (response.status) { - tableOptions.value = (response.data?.columns || []) - .filter((item: any) => item.type_name === 'table') - .map((item: any) => ({ label: item.object_name, value: item.object_name })) - } - }) - } - - watch(sourceCode, () => loadDatabases()) - watch(databaseName, () => loadTables()) - - onMounted(() => { - SourceService.getAll(new FilterModel()) - .then(response => { - if (response.status) { - sourceOptions.value = (response.data.content || []) - .map((item: any) => ({ label: item.name, value: item.code })) - } - }) - }) - - return { sourceCode, databaseName, tableName, sourceOptions, databaseOptions, tableOptions } -} diff --git a/core/metaforge-web/src/composables/useSourceDatabase.ts b/core/metaforge-web/src/composables/useSourceDatabase.ts index 0b22da09ca..0bfaeeb292 100644 --- a/core/metaforge-web/src/composables/useSourceDatabase.ts +++ b/core/metaforge-web/src/composables/useSourceDatabase.ts @@ -4,7 +4,7 @@ import { useI18n } from 'vue-i18n' import { MfMessage } from 'metaforge-ui' import MetadataService from '@/services/metadata' import SourceService from '@/services/source' -import { importLocation, sourceManageLocation } from '@/model/source-manage' +import { exportLocation, importLocation, sourceManageLocation } from '@/model/source-manage' import { mapDatabaseOverview, mapDatabaseTables, spaceSegments, type DatabaseOverview, type DatabaseTable } from '@/model/source-database' import { defaultCollation, mapCollations, type Collation } from '@/model/source-table-properties' import { useWorkspaceLoading } from './useWorkspaceLoading' @@ -91,7 +91,7 @@ export function useSourceDatabase() { if (action === 'create') tableCreateVisible.value = true else if (action === 'sql') router.push(sourceManageLocation({ source, database, tab: 'sql' })) else if (action === 'import') router.push(importLocation({ source, database })) - else router.push({ path: '/export', query: { source, database } }) + else router.push(exportLocation({ source, database })) } function onTableCreated() { tableCreateVisible.value = false diff --git a/core/metaforge-web/src/composables/useWorkbenchHeader.ts b/core/metaforge-web/src/composables/useWorkbenchHeader.ts index c48cd8c627..9f86777509 100644 --- a/core/metaforge-web/src/composables/useWorkbenchHeader.ts +++ b/core/metaforge-web/src/composables/useWorkbenchHeader.ts @@ -1,7 +1,7 @@ import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue' import { useSourceManageRoute as useRoute } from '@/composables/useSourceManageRoute' import { MfMessage } from 'metaforge-ui' -import { canOpenWorkbenchPage, compareLocation, erLocation, importLocation, sourceManageLocation } from '@/model/source-manage' +import { canOpenWorkbenchPage, compareLocation, erLocation, exportLocation, importLocation, sourceManageLocation } from '@/model/source-manage' import { useSourceCapabilities } from './useSourceCapabilities' import { useUserStore } from '@/stores/user' import { useI18n } from 'vue-i18n' @@ -29,7 +29,7 @@ export function useWorkbenchHeader() { { path: '/home', label: t('common.home'), icon: Home }, { path: '/manage', label: t('navigation.console'), icon: SquareTerminal, to: routeSource.value ? sourceManageLocation(context()) : '/source' }, { path: '/import', label: t('navigation.dataImport'), icon: Upload, to: routeSource.value ? importLocation(context()) : '/import' }, - { path: '/export', label: t('navigation.dataExport'), icon: Download }, + { path: '/export', label: t('navigation.dataExport'), icon: Download, to: routeSource.value ? exportLocation(context()) : '/export' }, { path: '/compare', label: t('navigation.dataCompare'), icon: GitCompare, to: routeSource.value ? compareLocation(context()) : '/compare' }, { path: '/er', label: t('navigation.erDiagram'), icon: Network, to: routeSource.value ? erLocation(context()) : '/er' }, ].filter(item => item.path === '/home' || canOpenWorkbenchPage(item.path, profile.value?.capabilities || []))) @@ -63,7 +63,8 @@ export function useWorkbenchHeader() { router.push(route.path === '/compare' ? compareLocation({ source: routeSource.value, database: value, mode: route.query.mode === 'data' ? 'data' : 'structure' }) : route.path === '/er' ? erLocation({ source: routeSource.value, database: value }) - : route.path === '/import' ? importLocation({ source: routeSource.value, database: value }) : sourceManageLocation({ source: routeSource.value, database: value })) + : route.path === '/import' ? importLocation({ source: routeSource.value, database: value }) + : route.path === '/export' ? exportLocation({ source: routeSource.value, database: value }) : sourceManageLocation({ source: routeSource.value, database: value })) } } }) diff --git a/core/metaforge-web/src/i18n/locales/en.json b/core/metaforge-web/src/i18n/locales/en.json index 038c8799d5..b201d40495 100644 --- a/core/metaforge-web/src/i18n/locales/en.json +++ b/core/metaforge-web/src/i18n/locales/en.json @@ -676,6 +676,213 @@ "er.find": "Find a table (Enter)", "er.fit": "Fit", "er.summary": "{database} · {tables} tables · {relations} foreign keys", + "export.cancel": "Cancel", + "export.chooseDatabase": "Choose a database at the top first", + "export.content.addCondition": "Add condition", + "export.content.all": "Structure and data", + "export.content.allHint": "CREATE statements go to schema.sql, data to one file per table", + "export.content.allRows": "All rows", + "export.content.allRowsHint": "Export every row of the table", + "export.content.conditions": "Conditions", + "export.content.conditionsHint": "Conditions are combined with AND", + "export.content.configured": "{done}/{total} configured", + "export.content.configuredTag": "Configured", + "export.content.create": "Include CREATE statements", + "export.content.currentTable": "Current table", + "export.content.currentTag": "Current", + "export.content.data": "Data only", + "export.content.dataHint": "Export only the rows", + "export.content.defaultTag": "Default", + "export.content.drop": "Include DROP statements", + "export.content.everyRow": "Every row", + "export.content.fields": "Fields", + "export.content.fieldsHint": "Unchecked fields are not exported", + "export.content.filtered": "Filtered", + "export.content.filteredHint": "Export only rows that match the conditions", + "export.content.hasFilter": "Filtered", + "export.content.indexes": "Include indexes", + "export.content.list": "Tables", + "export.content.noCondition": "No conditions", + "export.content.perTable": "Per table", + "export.content.range": "Rows", + "export.content.remove": "Remove", + "export.content.structure": "Structure", + "export.content.structureNote": "This source can not give table statements, so only data is exported.", + "export.content.title": "Content", + "export.content.value": "Value", + "export.content.what": "What to export", + "export.current": "Current: {database} · {count} tables chosen", + "export.currentDatabase": "Current database", + "export.databaseTag": "Database: {database}", + "export.done": "Export finished", + "export.download": "Download file", + "export.failed": "Export failed", + "export.finish": "Finish", + "export.format.chosen": "Chosen", + "export.format.copy": "Copy", + "export.format.count": "{count} installed", + "export.format.extension": "Extension .{extension}", + "export.format.kind.csv": "Comma separated, for spreadsheets", + "export.format.kind.json": "Structured data for programs", + "export.format.kind.other": "A format from a plugin", + "export.format.kind.txt": "Plain text split by a delimiter", + "export.format.kind.xml": "Markup for system exchange", + "export.format.live": "Live", + "export.format.noOptions": "This plugin has no settings", + "export.format.none": "No format plugin installed", + "export.format.note": "The preview is the first 3 rows of the first table, written by the plugin on the server.", + "export.format.options": "{name} settings", + "export.format.optionsHint": "Settings are declared by the plugin; the preview updates as they change", + "export.format.output": "Output", + "export.format.plugin": "Plugin", + "export.format.preview": "Output preview", + "export.format.refresh": "Refresh", + "export.format.search": "Search plugins", + "export.format.title": "Format plugins", + "export.format.version": "Version", + "export.next": "Next", + "export.number.1": "Step 1", + "export.number.2": "Step 2", + "export.number.3": "Step 3", + "export.number.4": "Step 4", + "export.number.5": "Step 5", + "export.objects.chosen": "Chosen objects", + "export.objects.clear": "Clear", + "export.objects.column.comment": "Comment", + "export.objects.column.name": "Name", + "export.objects.column.rows": "Rows", + "export.objects.column.size": "Size", + "export.objects.column.type": "Type", + "export.objects.database": "Whole database", + "export.objects.databaseHint": "Export every table and view of this database", + "export.objects.formatLater": "Chosen later", + "export.objects.kind.table": "Tables", + "export.objects.kind.view": "Views", + "export.objects.none": "Nothing chosen yet", + "export.objects.note": "Rows and sizes come from the source statistics and are estimates.", + "export.objects.search": "Search tables", + "export.objects.selectAll": "Select all", + "export.objects.selected": "{count} chosen", + "export.objects.summary": "Summary", + "export.objects.summaryRow.available": "Available", + "export.objects.summaryRow.chosen": "Chosen", + "export.objects.summaryRow.database": "Database", + "export.objects.summaryRow.format": "Format", + "export.objects.tables": "Choose tables", + "export.objects.tablesHint": "Choose from the {count} objects of {database}", + "export.objects.tablesUnit": "{count}", + "export.objects.title": "Objects", + "export.objects.type.table": "Table", + "export.objects.type.view": "View", + "export.operator.eq": "Equals", + "export.operator.ge": "At least", + "export.operator.gt": "Greater than", + "export.operator.isNull": "Is null", + "export.operator.le": "At most", + "export.operator.like": "Like", + "export.operator.lt": "Less than", + "export.operator.ne": "Not equal", + "export.operator.notLike": "Not like", + "export.operator.notNull": "Is not null", + "export.output.batchSize": "Batch size", + "export.output.browser": "Browser download", + "export.output.browserHint": "The browser downloads the file when the export is done", + "export.output.compress": "Compress", + "export.output.compressHint": "Output file: {file}", + "export.output.concurrency": "Concurrency", + "export.output.consistency": "Consistency and records", + "export.output.download": "Download", + "export.output.execution": "Execution and consistency", + "export.output.extension": "Extension", + "export.output.extensionValue": ".{extension} (set by {plugin})", + "export.output.failure": "On failure", + "export.output.failureOption.skip": "Skip the table", + "export.output.failureOption.stop": "Stop", + "export.output.file": "File and download", + "export.output.fileName": "File name", + "export.output.merge": "One merged file", + "export.output.mergeHint": "Every table goes into one file", + "export.output.mergeUnsupported": "{plugin} can not merge tables", + "export.output.note": "Data may still change during the export; use a smaller batch size for large tables.", + "export.output.organize": "Several tables", + "export.output.parallel": "{count} tables at once", + "export.output.reading": "Reading", + "export.output.readingOption.batch": "In batches", + "export.output.readingOption.single": "All at once", + "export.output.serial": "Serial", + "export.output.switch.autoDownload": "Download when done", + "export.output.switch.keepRecord": "Keep the export", + "export.output.switch.snapshot": "Consistent snapshot", + "export.output.switchHint.autoDownload": "Download the file as soon as it is ready", + "export.output.switchHint.keepRecord": "Keep the file for 2 hours after download", + "export.output.switchHint.snapshot": "Not supported yet: tables are read separately", + "export.output.zip": "One file per table", + "export.output.zipHint": "{count} {extension} files and schema.sql packed into a ZIP", + "export.output.zipHintData": "{count} {extension} files packed into a ZIP", + "export.preview.blocked": "{count} tables failed the check; go back and fix them", + "export.preview.column.data": "Data", + "export.preview.column.file": "File", + "export.preview.column.object": "Object", + "export.preview.column.rule": "Rows", + "export.preview.column.status": "Status", + "export.preview.column.structure": "Structure", + "export.preview.confirm": "Confirm", + "export.preview.download": "Download {file}", + "export.preview.full": "Full preview", + "export.preview.fullTitle": "Full preview: {file}", + "export.preview.jobStatus.done": "Done", + "export.preview.jobStatus.failed": "Failed", + "export.preview.jobStatus.partial": "Partly done", + "export.preview.jobStatus.running": "Running", + "export.preview.list": "Export list", + "export.preview.no": "No", + "export.preview.objectCount": "{count}", + "export.preview.ready": "Checked, {file} will be written", + "export.preview.rule.all": "All rows", + "export.preview.rule.filtered": "Filtered", + "export.preview.sample": "Sample rows: {table} ({count})", + "export.preview.sampleNote": "The sample shows the first 3 raw rows; the file is written in the chosen format.", + "export.preview.status.error": "Error", + "export.preview.status.pass": "Ready", + "export.preview.summary.compress": "Compress", + "export.preview.summary.concurrency": "Concurrency", + "export.preview.summary.content": "Content", + "export.preview.summary.database": "Database", + "export.preview.summary.failure": "On failure", + "export.preview.summary.format": "Format", + "export.preview.summary.objects": "Objects", + "export.preview.summary.output": "Output", + "export.preview.tableStatus.cancelled": "Cancelled", + "export.preview.tableStatus.done": "Done", + "export.preview.tableStatus.failed": "Failed", + "export.preview.tableStatus.running": "Running", + "export.preview.tableStatus.waiting": "Waiting", + "export.preview.tile.errors": "Errors", + "export.preview.tile.files": "Files", + "export.preview.tile.rows": "Estimated rows", + "export.preview.tile.tables": "Objects", + "export.preview.written": "{count} rows written", + "export.preview.yes": "Yes", + "export.previous": "Previous", + "export.start": "Start export", + "export.step.content": "Set content", + "export.step.format": "Format plugin", + "export.step.objects": "Choose objects", + "export.step.output": "Output", + "export.step.preview": "Preview and export", + "export.stepDescription.content": "Choose structure or data, then set fields and filters per table.", + "export.stepDescription.format": "The format plugin decides the file format; its settings are declared by the plugin.", + "export.stepDescription.objects": "Choose the tables or views to export, or the whole database.", + "export.stepDescription.output": "Set the file name, how tables are packed and how the export runs.", + "export.stepDescription.preview": "Check the export list and sample rows, then start the export.", + "export.stepHeading.content": "Set what each table exports", + "export.stepHeading.format": "Choose a format plugin", + "export.stepHeading.objects": "Choose what to export", + "export.stepHeading.output": "Set the output", + "export.stepHeading.preview": "Preview and export", + "export.stepTitle": "{index}. {title}", + "export.subtitle": "Choose tables, set content and format, and download them as files", + "export.tableCount": "{count} tables", "function.common.create": "Create Function", "function.common.delete": "Delete function", "function.common.deleteInfo": "Delete function [ $VALUE ]", @@ -2222,11 +2429,6 @@ "template.common.modify": "Modify Template [ $NAME ]", "template.tip.createSuccess": "Template [ $VALUE ] saved successfully", "tools.compare.title": "Data compare", - "tools.export.count": "Rows", - "tools.export.countAll": "Leave empty to export every row", - "tools.export.desc": "Pick a source and table, choose a format and export for download.", - "tools.export.execute": "Export", - "tools.export.title": "Data export", "user.auth.captchaIsNumberTip": "Captcha must be a number", "user.auth.captchaSizeTip": "Captcha must be 1-6 characters long", "user.auth.captchaTip": "Please enter captcha", diff --git a/core/metaforge-web/src/i18n/locales/zh-cn.json b/core/metaforge-web/src/i18n/locales/zh-cn.json index 3629bc7c58..015ed7a685 100644 --- a/core/metaforge-web/src/i18n/locales/zh-cn.json +++ b/core/metaforge-web/src/i18n/locales/zh-cn.json @@ -676,6 +676,213 @@ "er.find": "定位数据表（回车）", "er.fit": "适应画布", "er.summary": "{database} · {tables} 张表 · {relations} 个外键关系", + "export.cancel": "取消", + "export.chooseDatabase": "请先在顶部选择数据库", + "export.content.addCondition": "添加条件", + "export.content.all": "结构和数据", + "export.content.allHint": "建表语句写入 schema.sql，数据按表写入文件", + "export.content.allRows": "全部数据", + "export.content.allRowsHint": "导出表中的所有行", + "export.content.conditions": "筛选条件", + "export.content.conditionsHint": "多个条件之间为 AND 关系", + "export.content.configured": "已配置 {done}/{total}", + "export.content.configuredTag": "已配置", + "export.content.create": "包含建表语句", + "export.content.currentTable": "当前表", + "export.content.currentTag": "当前", + "export.content.data": "仅数据", + "export.content.dataHint": "只导出表中的数据", + "export.content.defaultTag": "默认", + "export.content.drop": "包含 DROP 语句", + "export.content.everyRow": "全部行", + "export.content.fields": "导出字段", + "export.content.fieldsHint": "取消勾选的字段不会导出", + "export.content.filtered": "按条件筛选", + "export.content.filteredHint": "只导出满足条件的行", + "export.content.hasFilter": "已筛选", + "export.content.indexes": "包含索引", + "export.content.list": "导出表", + "export.content.noCondition": "暂无条件", + "export.content.perTable": "按表设置", + "export.content.range": "数据范围", + "export.content.remove": "删除", + "export.content.structure": "结构选项", + "export.content.structureNote": "当前数据源不支持导出表结构，只能导出数据。", + "export.content.title": "导出内容", + "export.content.value": "值", + "export.content.what": "导出什么", + "export.current": "当前：{database} · 已选 {count} 张表", + "export.currentDatabase": "当前数据库", + "export.databaseTag": "数据库：{database}", + "export.done": "导出完成", + "export.download": "下载文件", + "export.failed": "导出失败", + "export.finish": "完成", + "export.format.chosen": "已选", + "export.format.copy": "复制", + "export.format.count": "已安装 {count}", + "export.format.extension": "扩展名 .{extension}", + "export.format.kind.csv": "逗号分隔，适合表格软件", + "export.format.kind.json": "结构化数据，适合程序读取", + "export.format.kind.other": "由插件提供的格式", + "export.format.kind.txt": "纯文本，按分隔符分列", + "export.format.kind.xml": "标记语言，适合系统交换", + "export.format.live": "实时", + "export.format.noOptions": "该插件没有可配置的参数", + "export.format.none": "没有可用的格式化插件", + "export.format.note": "预览使用第一张表的前 3 行，由插件在服务器上生成。", + "export.format.options": "{name} 参数", + "export.format.optionsHint": "参数由插件声明，修改后右侧预览即时更新", + "export.format.output": "输出格式", + "export.format.plugin": "已选插件", + "export.format.preview": "输出预览", + "export.format.refresh": "刷新", + "export.format.search": "搜索插件", + "export.format.title": "格式化插件", + "export.format.version": "版本", + "export.next": "下一步", + "export.number.1": "一", + "export.number.2": "二", + "export.number.3": "三", + "export.number.4": "四", + "export.number.5": "五", + "export.objects.chosen": "已选对象", + "export.objects.clear": "清空", + "export.objects.column.comment": "注释", + "export.objects.column.name": "名称", + "export.objects.column.rows": "行数", + "export.objects.column.size": "大小", + "export.objects.column.type": "类型", + "export.objects.database": "整个数据库", + "export.objects.databaseHint": "导出当前数据库的全部表和视图", + "export.objects.formatLater": "下一步选择", + "export.objects.kind.table": "表", + "export.objects.kind.view": "视图", + "export.objects.none": "尚未选择任何对象", + "export.objects.note": "行数与大小来自数据源统计信息，仅供参考。", + "export.objects.search": "搜索表名", + "export.objects.selectAll": "全选", + "export.objects.selected": "已选 {count}", + "export.objects.summary": "导出概览", + "export.objects.summaryRow.available": "可选对象", + "export.objects.summaryRow.chosen": "已选对象", + "export.objects.summaryRow.database": "数据库", + "export.objects.summaryRow.format": "导出格式", + "export.objects.tables": "选择表", + "export.objects.tablesHint": "从 {database} 的 {count} 个对象中选择", + "export.objects.tablesUnit": "{count} 个", + "export.objects.title": "导出对象", + "export.objects.type.table": "表", + "export.objects.type.view": "视图", + "export.operator.eq": "等于", + "export.operator.ge": "大于等于", + "export.operator.gt": "大于", + "export.operator.isNull": "为空", + "export.operator.le": "小于等于", + "export.operator.like": "包含", + "export.operator.lt": "小于", + "export.operator.ne": "不等于", + "export.operator.notLike": "不包含", + "export.operator.notNull": "不为空", + "export.output.batchSize": "批大小", + "export.output.browser": "浏览器下载", + "export.output.browserHint": "导出完成后由浏览器下载文件", + "export.output.compress": "压缩", + "export.output.compressHint": "输出文件：{file}", + "export.output.concurrency": "并发", + "export.output.consistency": "一致性与记录", + "export.output.download": "下载方式", + "export.output.execution": "执行与一致性", + "export.output.extension": "扩展名", + "export.output.extensionValue": ".{extension}（由 {plugin} 决定）", + "export.output.failure": "失败处理", + "export.output.failureOption.skip": "跳过该表", + "export.output.failureOption.stop": "停止导出", + "export.output.file": "文件与下载设置", + "export.output.fileName": "文件名", + "export.output.merge": "合并为一个文件", + "export.output.mergeHint": "所有表写入同一个文件", + "export.output.mergeUnsupported": "{plugin} 不支持合并多张表", + "export.output.note": "导出期间数据仍可能被修改，大表请适当调小批大小。", + "export.output.organize": "多表文件组织", + "export.output.parallel": "{count} 张表并行", + "export.output.reading": "读取方式", + "export.output.readingOption.batch": "分批读取", + "export.output.readingOption.single": "一次读取", + "export.output.serial": "串行", + "export.output.switch.autoDownload": "完成后自动下载", + "export.output.switch.keepRecord": "保留导出记录", + "export.output.switch.snapshot": "一致性快照", + "export.output.switchHint.autoDownload": "导出完成后立即下载文件", + "export.output.switchHint.keepRecord": "下载后保留文件 2 小时，可再次下载", + "export.output.switchHint.snapshot": "暂不支持：各表分别读取", + "export.output.zip": "每张表一个文件", + "export.output.zipHint": "{count} 个 {extension} 文件与 schema.sql 打包为 ZIP", + "export.output.zipHintData": "{count} 个 {extension} 文件打包为 ZIP", + "export.preview.blocked": "{count} 张表校验失败，请返回修改", + "export.preview.column.data": "数据", + "export.preview.column.file": "文件", + "export.preview.column.object": "对象", + "export.preview.column.rule": "规则", + "export.preview.column.status": "状态", + "export.preview.column.structure": "结构", + "export.preview.confirm": "导出确认", + "export.preview.download": "下载 {file}", + "export.preview.full": "查看完整预览", + "export.preview.fullTitle": "完整预览：{file}", + "export.preview.jobStatus.done": "已完成", + "export.preview.jobStatus.failed": "失败", + "export.preview.jobStatus.partial": "部分完成", + "export.preview.jobStatus.running": "导出中", + "export.preview.list": "最终导出清单", + "export.preview.no": "否", + "export.preview.objectCount": "{count} 个", + "export.preview.ready": "校验通过，将生成 {file}", + "export.preview.rule.all": "全部数据", + "export.preview.rule.filtered": "按条件", + "export.preview.sample": "示例数据预览：{table}（{count} 行）", + "export.preview.sampleNote": "示例数据只展示前 3 行原始值，实际文件按所选格式写出。", + "export.preview.status.error": "错误", + "export.preview.status.pass": "通过", + "export.preview.summary.compress": "压缩", + "export.preview.summary.concurrency": "并发", + "export.preview.summary.content": "导出内容", + "export.preview.summary.database": "数据库", + "export.preview.summary.failure": "失败处理", + "export.preview.summary.format": "格式", + "export.preview.summary.objects": "导出对象", + "export.preview.summary.output": "输出文件", + "export.preview.tableStatus.cancelled": "已取消", + "export.preview.tableStatus.done": "已完成", + "export.preview.tableStatus.failed": "失败", + "export.preview.tableStatus.running": "导出中", + "export.preview.tableStatus.waiting": "等待中", + "export.preview.tile.errors": "校验错误", + "export.preview.tile.files": "输出文件", + "export.preview.tile.rows": "预计行数", + "export.preview.tile.tables": "导出对象", + "export.preview.written": "已写入 {count} 行", + "export.preview.yes": "是", + "export.previous": "上一步", + "export.start": "开始导出", + "export.step.content": "配置导出内容", + "export.step.format": "格式化插件", + "export.step.objects": "选择导出对象", + "export.step.output": "输出设置", + "export.step.preview": "预览并导出", + "export.stepDescription.content": "选择导出结构或数据，并按表设置字段与筛选条件。", + "export.stepDescription.format": "格式化插件决定文件格式，参数由插件声明。", + "export.stepDescription.objects": "选择要导出的表或视图，也可以导出整个数据库。", + "export.stepDescription.output": "设置文件名、多表组织方式以及执行策略。", + "export.stepDescription.preview": "确认导出清单与示例数据，然后开始导出。", + "export.stepHeading.content": "配置导出内容", + "export.stepHeading.format": "选择格式化插件", + "export.stepHeading.objects": "选择导出对象", + "export.stepHeading.output": "输出设置", + "export.stepHeading.preview": "预览并导出", + "export.stepTitle": "{index}、{title}", + "export.subtitle": "选择表、配置内容与格式，导出为文件下载", + "export.tableCount": "{count} 张表", "function.common.create": "创建函数", "function.common.delete": "删除函数", "function.common.deleteInfo": "删除函数 [ $VALUE ]", @@ -2222,11 +2429,6 @@ "template.common.modify": "修改模板 [ $NAME ]", "template.tip.createSuccess": "模板 [ $VALUE ] 保存成功", "tools.compare.title": "数据对比", - "tools.export.count": "导出条数", - "tools.export.countAll": "留空导出全部数据", - "tools.export.desc": "选择数据源与数据表，选择格式后导出下载。", - "tools.export.execute": "开始导出", - "tools.export.title": "数据导出", "user.auth.captchaIsNumberTip": "验证码必须为数字", "user.auth.captchaSizeTip": "验证码必须在1-6个字符之间", "user.auth.captchaTip": "请输入验证码", diff --git a/core/metaforge-web/src/model/database-export.ts b/core/metaforge-web/src/model/database-export.ts new file mode 100644 index 0000000000..f93ace6ce2 --- /dev/null +++ b/core/metaforge-web/src/model/database-export.ts @@ -0,0 +1,100 @@ +/** One table of the database as the export lists it. */ +export interface ExportObject { name: string; type: 'table' | 'view'; rows: number | null; bytes: number | null; comment: string } +export interface ExportColumn { name: string; type: string } +export interface ExportFilter { column: string; operator: string; value: string } +export interface TableSetting { fields: string[]; filters: ExportFilter[]; scope: 'all' | 'filtered' } +export interface FormatChoice { value: string; labels: Record } +export interface FormatOption { name: string; labels: Record; defaultValue: string; choices: FormatChoice[] } +export interface ExportFormat { name: string; version: string; extension: string; merge: boolean; options: FormatOption[] } + +export interface ExportState { + objects: 'tables' | 'database' + selected: string[] + content: 'all' | 'data' + structure: { create: boolean; indexes: boolean; drop: boolean } + settings: Record + format: string + options: Record + fileName: string + organize: 'zip' | 'merge' + compress: boolean + batchSize: number + concurrency: number + reading: 'batch' | 'single' + failure: 'stop' | 'skip' + autoDownload: boolean + keepRecord: boolean +} + +export const EXPORT_STEPS = ['objects', 'content', 'format', 'output', 'preview'] as const +export const OPERATORS = ['=', '!=', '>', '>=', '<', '<=', 'LIKE', 'NOT LIKE', 'IS NULL', 'IS NOT NULL'] as const +export const needsValue = (operator: string) => !operator.startsWith('IS ') + +const pad = (value: number) => String(value).padStart(2, '0') +/** database_export_yyyyMMdd */ +export const defaultFileName = (database: string, today = new Date()) => `${ database || 'export' }_export_${ today.getFullYear() }${ pad(today.getMonth() + 1) }${ pad(today.getDate()) }` + +export function defaultExportState(database: string): ExportState { + return { + objects: 'tables', selected: [], content: 'all', structure: { create: true, indexes: true, drop: false }, settings: {}, + format: '', options: {}, fileName: defaultFileName(database), organize: 'zip', compress: true, batchSize: 1000, concurrency: 1, + reading: 'batch', failure: 'stop', autoDownload: true, keepRecord: true, + } +} + +/** The tables the export writes: every listed object for a whole-database export, the checked ones otherwise. */ +export const exportedTables = (state: ExportState, objects: ExportObject[]) => state.objects === 'database' ? objects.map(object => object.name) : state.selected + +export const setting = (state: ExportState, table: string): TableSetting => state.settings[table] || { fields: [], filters: [], scope: 'all' } + +/** A table counts as configured once its fields or rows differ from "every field, every row". */ +export const configured = (value: TableSetting | undefined) => !!value && (value.fields.length > 0 || (value.scope === 'filtered' && value.filters.length > 0)) + +/** The format's declared options with their defaults, keeping values already chosen. */ +export function formatOptions(format: ExportFormat | undefined, current: Record): Record { + const values: Record = {} + for (const option of format?.options || []) values[option.name] = current[option.name] ?? option.defaultValue + return values +} + +/** The plan the server checks and runs. Fields equal to every column are sent as "all" (empty). */ +export function exportPlan(state: ExportState, objects: ExportObject[], columns: Record) { + return { + tables: exportedTables(state, objects).map(name => { + const value = setting(state, name) + const all = columns[name]?.length ?? -1 + return { + name, + fields: value.fields.length === all ? [] : [...value.fields], + filters: value.scope === 'filtered' ? value.filters.filter(filter => filter.column && (!needsValue(filter.operator) || filter.value !== '')) : [], + } + }), + content: state.content, + structure: { ...state.structure }, + format: state.format, + options: { ...state.options }, + fileName: state.fileName, + organize: state.organize, + compress: state.compress, + batchSize: state.batchSize, + concurrency: state.concurrency, + reading: state.reading, + failure: state.failure, + keepRecord: state.keepRecord, + } +} + +/** The download: one file as written when it is the only one and not compressed, a ZIP otherwise (as the server names it). */ +export function outputName(state: ExportState, tables: string[], extension: string) { + const files = tables.length + (state.content === 'all' && (state.structure.create || state.structure.drop) ? 1 : 0) + const base = (state.fileName || '').trim().replace(/[\\/:*?"<>|\s]+/g, '_') || 'export' + return files === 1 && !state.compress ? `${ tables[0] }.${ extension }` : `${ base }.zip` +} + +/** A label of a plugin option or choice in the page's language. */ +export const label = (labels: Record, locale: string) => labels[locale.startsWith('zh') ? 'zh' : 'en'] || Object.values(labels)[0] || '' + +export function formatBytes(bytes: number | null) { + if (bytes === null || bytes === undefined) return '-' + return `${ (bytes / 1024 / 1024).toFixed(2) } MB` +} diff --git a/core/metaforge-web/src/model/source-manage.ts b/core/metaforge-web/src/model/source-manage.ts index f25b58b9bb..691721d611 100644 --- a/core/metaforge-web/src/model/source-manage.ts +++ b/core/metaforge-web/src/model/source-manage.ts @@ -5,7 +5,7 @@ export function sourceManageContext(route: { path?: string; params: Record = {}) { @@ -29,18 +29,19 @@ export function compareLocation(context: { source: string; database?: string; ta return { path: '/compare', query } } -/** A workbench page of a source's database: the ER diagram or the import. */ -function databasePage(path: '/er' | '/import', context: { source: string; database?: string }) { +/** A workbench page of a source's database: the ER diagram, the import or the export. */ +function databasePage(path: '/er' | '/import' | '/export', context: { source: string; database?: string }) { const query: Record = { id: context.source } if (context.database) query.database = context.database return { path, query } } export const erLocation = (context: { source: string; database?: string }) => databasePage('/er', context) export const importLocation = (context: { source: string; database?: string }) => databasePage('/import', context) +export const exportLocation = (context: { source: string; database?: string }) => databasePage('/export', context) /** Pages showing the source workbench header: the workspace, and the compare and ER pages of a source. */ export function isSourceWorkbenchRoute(route: { path: string; query: Record }): boolean { - if (route.path === '/compare' || route.path === '/er' || route.path === '/import') return !!sourceManageContext({ path: route.path, params: {}, query: route.query }).source + if (route.path === '/compare' || route.path === '/er' || route.path === '/import' || route.path === '/export') return !!sourceManageContext({ path: route.path, params: {}, query: route.query }).source return isSourceManagementRoute(route) } @@ -56,7 +57,7 @@ export function isSourceManagementRoute(route: { path: string; query: Record = { '/manage': [], '/import': ['data:insert', 'table:list', 'column:read'], - '/export': ['table:export'], + '/export': ['table:export', 'table:list', 'column:read'], '/compare': ['table:list', 'column:read'], '/er': ['database:info', 'tab:er-diagram'], } diff --git a/core/metaforge-web/src/services/export.ts b/core/metaforge-web/src/services/export.ts new file mode 100644 index 0000000000..f7ee8901f9 --- /dev/null +++ b/core/metaforge-web/src/services/export.ts @@ -0,0 +1,40 @@ +import { HttpUtils } from '@/utils/http' +import { ResponseModel } from '@/model/response' + +const DEFAULT_PATH = '/api/v1/export' + +/** Export of a database's tables: formats, samples, the plan check, the job and its file. */ +class ExportService +{ + formats(): Promise + { + return new HttpUtils().get(`${ DEFAULT_PATH }/formats`) + } + + sample(code: string, database: string, plan: any, table: string, rows = 3): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/${ encodeURIComponent(code) }/${ encodeURIComponent(database) }/sample?table=${ encodeURIComponent(table) }&rows=${ rows }`, plan) + } + + check(code: string, database: string, plan: any): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/${ encodeURIComponent(code) }/${ encodeURIComponent(database) }/check`, plan) + } + + execute(code: string, database: string, plan: any): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/${ encodeURIComponent(code) }/${ encodeURIComponent(database) }/execute`, plan) + } + + job(id: string): Promise + { + return new HttpUtils().get(`${ DEFAULT_PATH }/jobs/${ id }`) + } + + file(id: string) + { + return new HttpUtils().download(`${ DEFAULT_PATH }/jobs/${ id }/file`) + } +} + +export default new ExportService() diff --git a/core/metaforge-web/src/views/pages/tools/ExportHome.vue b/core/metaforge-web/src/views/pages/tools/ExportHome.vue index 951f673a63..d5c20578fa 100644 --- a/core/metaforge-web/src/views/pages/tools/ExportHome.vue +++ b/core/metaforge-web/src/views/pages/tools/ExportHome.vue @@ -1,52 +1,80 @@ diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportChoice.vue b/core/metaforge-web/src/views/pages/tools/export/ExportChoice.vue new file mode 100644 index 0000000000..9d5026ab37 --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportChoice.vue @@ -0,0 +1,20 @@ + + + diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportStepContent.vue b/core/metaforge-web/src/views/pages/tools/export/ExportStepContent.vue new file mode 100644 index 0000000000..c2295c0a23 --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportStepContent.vue @@ -0,0 +1,98 @@ + + + diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportStepFormat.vue b/core/metaforge-web/src/views/pages/tools/export/ExportStepFormat.vue new file mode 100644 index 0000000000..22b62dea9e --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportStepFormat.vue @@ -0,0 +1,100 @@ + + + diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportStepObjects.vue b/core/metaforge-web/src/views/pages/tools/export/ExportStepObjects.vue new file mode 100644 index 0000000000..8dbb4533dc --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportStepObjects.vue @@ -0,0 +1,94 @@ + + + diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportStepOutput.vue b/core/metaforge-web/src/views/pages/tools/export/ExportStepOutput.vue new file mode 100644 index 0000000000..bc7cea516b --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportStepOutput.vue @@ -0,0 +1,70 @@ + + + diff --git a/core/metaforge-web/src/views/pages/tools/export/ExportStepPreview.vue b/core/metaforge-web/src/views/pages/tools/export/ExportStepPreview.vue new file mode 100644 index 0000000000..74c7462be8 --- /dev/null +++ b/core/metaforge-web/src/views/pages/tools/export/ExportStepPreview.vue @@ -0,0 +1,128 @@ + + + diff --git a/core/metaforge-web/tests/database-export.test.mjs b/core/metaforge-web/tests/database-export.test.mjs new file mode 100644 index 0000000000..3cd5a6fbc2 --- /dev/null +++ b/core/metaforge-web/tests/database-export.test.mjs @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' +import ts from 'typescript' +const source = await readFile(new URL('../src/model/database-export.ts', import.meta.url), 'utf8') +const { outputText } = ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2020 } }) +const { defaultExportState, defaultFileName, exportPlan, exportedTables, configured, formatOptions, outputName, label } = await import(`data:text/javascript;base64,${Buffer.from(outputText).toString('base64')}`) + +const objects = [{ name: 'chat', type: 'table', rows: 3, bytes: 1, comment: '' }, { name: 'logs', type: 'table', rows: 1, bytes: 1, comment: '' }] +const columns = { chat: [{ name: 'id', type: 'BIGINT' }, { name: 'title', type: 'VARCHAR' }], logs: [{ name: 'id', type: 'BIGINT' }] } + +test('the file name carries the database and the day', () => { + assert.equal(defaultFileName('app', new Date(2026, 9, 31)), 'app_export_20261031') +}) + +test('a whole-database export takes every object; a table selection only the checked ones', () => { + const state = defaultExportState('app') + state.selected = ['logs'] + assert.deepEqual(exportedTables(state, objects), ['logs']) + state.objects = 'database' + assert.deepEqual(exportedTables(state, objects), ['chat', 'logs']) +}) + +test('the plan sends every field as empty, keeps chosen fields and only complete filters', () => { + const state = defaultExportState('app') + state.selected = ['chat', 'logs'] + state.settings.chat = { fields: ['id'], scope: 'filtered', filters: [{ column: 'id', operator: '>=', value: '2' }, { column: 'title', operator: '=', value: '' }, { column: 'title', operator: 'IS NULL', value: '' }] } + state.settings.logs = { fields: ['id'], scope: 'all', filters: [{ column: 'id', operator: '=', value: '1' }] } + const plan = exportPlan(state, objects, columns) + assert.deepEqual(plan.tables[0], { name: 'chat', fields: ['id'], filters: [{ column: 'id', operator: '>=', value: '2' }, { column: 'title', operator: 'IS NULL', value: '' }] }) + assert.deepEqual(plan.tables[1], { name: 'logs', fields: [], filters: [] }) + assert.equal(configured(state.settings.chat), true) + assert.equal(configured({ fields: [], filters: [], scope: 'all' }), false) +}) + +test('format options start from the plugin defaults and keep chosen values', () => { + const format = { name: 'JsonConvert', options: [{ name: 'indent', labels: { zh: '缩进', en: 'Indent' }, defaultValue: '2', choices: [] }, { name: 'root', labels: {}, defaultValue: 'array', choices: [] }] } + assert.deepEqual(formatOptions(format, { indent: '4', stale: 'x' }), { indent: '4', root: 'array' }) + assert.equal(label({ zh: '缩进', en: 'Indent' }, 'zh-cn'), '缩进') + assert.equal(label({ zh: '缩进', en: 'Indent' }, 'en'), 'Indent') +}) + +test('the output is a ZIP unless one uncompressed file is all there is', () => { + const state = defaultExportState('app') + state.fileName = 'app export' + assert.equal(outputName(state, ['chat', 'logs'], 'json'), 'app_export.zip') + state.content = 'data' + state.compress = false + assert.equal(outputName(state, ['chat'], 'json'), 'chat.json') + state.compress = true + assert.equal(outputName(state, ['chat'], 'json'), 'app_export.zip') +}) From 058f928b001fbf2bf84535848f9c116b0b649d70 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:24:19 -0400 Subject: [PATCH 005/232] feat(export): check and run database exports as background jobs The export API lists installed formats with their declared settings, writes a sample of a table with the chosen format, checks a plan per table, and runs it as a job that reads tables in batches, writes one file per table plus schema.sql, and packs them into a ZIP to download. --- .../controller/DatabaseExportController.java | 82 +++ .../server/exporter/DatabaseExportTest.java | 278 ++++++++ .../service/exporter/ExportPlan.java | 69 ++ .../service/exporter/ExportStatements.java | 115 ++++ .../service/DatabaseExportService.java | 28 + .../impl/DatabaseExportServiceImpl.java | 621 ++++++++++++++++++ 6 files changed, 1193 insertions(+) create mode 100644 core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java create mode 100644 core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java new file mode 100644 index 0000000000..c0956ad70b --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/DatabaseExportController.java @@ -0,0 +1,82 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +/** Export of a database's tables: formats, samples, the plan check, the job and its file. */ +@RestController +@RequestMapping(value = "/api/v1/export") +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class DatabaseExportController +{ + private final DatabaseExportService service; + + public DatabaseExportController(DatabaseExportService service) + { + this.service = service; + } + + @GetMapping(value = "formats") + public CommonResponse>> formats() + { + return this.service.formats(); + } + + @PostMapping(value = "{code}/{database}/sample") + public CommonResponse> sample(@PathVariable String code, @PathVariable String database, @RequestParam String table, + @RequestParam(defaultValue = "3") int rows, @RequestBody ExportPlan plan) + { + return this.service.sample(code, database, plan, table, rows); + } + + @PostMapping(value = "{code}/{database}/check") + public CommonResponse> check(@PathVariable String code, @PathVariable String database, @RequestBody ExportPlan plan) + { + return this.service.check(code, database, plan); + } + + @PostMapping(value = "{code}/{database}/execute") + public CommonResponse> execute(@PathVariable String code, @PathVariable String database, @RequestBody ExportPlan plan) + { + return this.service.execute(code, database, plan); + } + + @GetMapping(value = "jobs/{id}") + public CommonResponse> job(@PathVariable String id) + { + return this.service.job(id); + } + + @PostMapping(value = "jobs/{id}/file") + public ResponseEntity file(@PathVariable String id) + { + CommonResponse response = this.service.file(id); + if (!Boolean.TRUE.equals(response.getStatus()) || response.getData() == null) { + return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(response); + } + TableExportService.ExportFile file = response.getData(); + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(file.getName(), StandardCharsets.UTF_8).build().toString()) + .header(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION) + .body(file.getContent()); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java new file mode 100644 index 0000000000..365bd03152 --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/exporter/DatabaseExportTest.java @@ -0,0 +1,278 @@ +package org.devlive.metaforge.server.exporter; + +import org.apache.commons.io.FileUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.convert.ConvertService; +import org.devlive.metaforge.convert.model.ConvertOption; +import org.devlive.metaforge.convert.model.ConvertRequest; +import org.devlive.metaforge.convert.model.ConvertResponse; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.exporter.ExportStatements; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.devlive.metaforge.service.service.impl.DatabaseExportServiceImpl; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.ManagementCapabilities; +import org.devlive.metaforge.spi.model.ManagementCapability; +import org.devlive.metaforge.spi.model.Response; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DatabaseExportTest +{ + private final ExportStatements statements = new ExportStatements(name -> "`" + name + "`", value -> "'" + value.replace("'", "''") + "'"); + + private static ExportPlan.Filter filter(String column, String operator, String value) + { + ExportPlan.Filter filter = new ExportPlan.Filter(); + filter.setColumn(column); + filter.setOperator(operator); + filter.setValue(value); + return filter; + } + + private static ExportPlan.TablePlan table(String name, List fields, ExportPlan.Filter... filters) + { + ExportPlan.TablePlan table = new ExportPlan.TablePlan(); + table.setName(name); + table.setFields(new ArrayList<>(fields)); + table.setFilters(new ArrayList<>(List.of(filters))); + return table; + } + + @Test + public void selectsTakeFieldsFiltersAndPages() + { + ExportPlan.TablePlan chat = table("chat", List.of("id", "title"), filter("created", ">=", "2026-10-01"), filter("title", "IS NOT NULL", null)); + assertEquals("SELECT `id`, `title` FROM `app`.`chat` WHERE `created` >= '2026-10-01' AND `title` IS NOT NULL LIMIT 1000 OFFSET 2000", + statements.select("app", chat, 1000, 2000)); + assertEquals("SELECT * FROM `app`.`logs`", statements.select("app", table("logs", List.of()), 0, 0)); + assertEquals("SELECT COUNT(*) FROM `app`.`chat` WHERE `title` LIKE 'it''s%'", statements.count("app", table("chat", List.of(), filter("title", "like", "it's%")))); + } + + @Test + public void theSchemaDropsCreatesAndCanLeaveIndexesOut() + { + String create = "CREATE TABLE `chat` (\n `id` bigint NOT NULL,\n `user_id` bigint,\n PRIMARY KEY (`id`),\n KEY `idx_user` (`user_id`),\n" + + " CONSTRAINT `fk_user` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`)\n) ENGINE=InnoDB"; + ExportPlan.Structure structure = new ExportPlan.Structure(); + structure.setDrop(true); + structure.setIndexes(false); + assertEquals("DROP TABLE IF EXISTS `app`.`chat`;\nCREATE TABLE `chat` (\n `id` bigint NOT NULL,\n `user_id` bigint,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB;\n", + statements.schema("app", "chat", create, structure)); + structure.setIndexes(true); + structure.setDrop(false); + assertTrue(statements.schema("app", "chat", create, structure).contains("KEY `idx_user`")); + } + + // ---- service ---- + + private final List executed = Collections.synchronizedList(new ArrayList<>()); + + /** A format writing "headers|rows|options" as a .fake file. */ + private static final class FakeConvert + implements ConvertService + { + @Override + public List options() + { + return List.of(new ConvertOption("indent", Map.of("en", "Indent"), "2", List.of())); + } + + @Override + public String extension() + { + return "fake"; + } + + @Override + public ConvertResponse format(ConvertRequest request) + { + return new ConvertResponse(); + } + + @Override + public ConvertResponse formatStream(ConvertRequest request) + { + return new ConvertResponse(); + } + + @Override + public ConvertResponse writer(ConvertRequest request) + { + ConvertResponse response = new ConvertResponse(); + try { + FileUtils.writeStringToFile(new File(request.getPath(), request.getName() + ".fake"), request.getHeaders() + "|" + request.getColumns() + "|" + request.getOptions(), + StandardCharsets.UTF_8); + response.setSuccessful(true); + } + catch (IOException error) { + response.setSuccessful(false); + } + return response; + } + + @Override + public ConvertResponse reader(ConvertRequest request) + { + return new ConvertResponse(); + } + } + + private Response answer(String sql) + { + executed.add(sql); + List rows = new ArrayList<>(); + if (sql.contains("information_schema.COLUMNS")) { + rows.add(List.of("chat", "id", "bigint")); + rows.add(List.of("chat", "title", "varchar(64)")); + rows.add(List.of("logs", "id", "bigint")); + return Response.builder().isSuccessful(true).headers(List.of("TABLE_NAME", "COLUMN_NAME", "COLUMN_TYPE")).columns(rows).build(); + } + if (sql.startsWith("SHOW CREATE TABLE")) { + String table = sql.contains("`chat`") ? "chat" : "logs"; + rows.add(List.of(table, "CREATE TABLE `" + table + "` (\n `id` bigint NOT NULL,\n KEY `idx` (`id`)\n)")); + return Response.builder().isSuccessful(true).headers(List.of("Table", "Create Table")).columns(rows).build(); + } + if (sql.startsWith("SELECT COUNT(*)")) { + rows.add(List.of(3)); + return Response.builder().isSuccessful(true).headers(List.of("COUNT(*)")).columns(rows).build(); + } + // Three rows, served a page at a time. + Matcher page = Pattern.compile("LIMIT (\\d+) OFFSET (\\d+)$").matcher(sql); + long limit = page.find() ? Long.parseLong(page.group(1)) : 3; + long offset = page.find(0) ? Long.parseLong(page.group(2)) : 0; + for (long id = offset + 1; id <= Math.min(3, offset + limit); id++) { + rows.add(List.of(id, "t" + id)); + } + return Response.builder().isSuccessful(true).headers(List.of("id", "title")).columns(rows).build(); + } + + private DatabaseExportService service() + { + SourceRepository repository = mock(SourceRepository.class); + PluginManager manager = mock(PluginManager.class); + Plugin source = mock(Plugin.class); + Plugin convert = mock(Plugin.class); + PluginService adapter = new PluginService() + { + @Override + public ManagementCapabilities managementCapabilities() + { + return new ManagementCapabilities("relational", ManagementCapability.keys()); + } + + @Override + public Response execute(RuntimeConfigure configure, String content) + { + return answer(content); + } + }; + SourceEntity entity = new SourceEntity(); + entity.setType("MySQL"); + when(repository.findByCode("demo")).thenReturn(Optional.of(entity)); + when(manager.getPlugin("MySQL")).thenReturn(Optional.of(source)); + when(manager.getPlugin("FakeConvert")).thenReturn(Optional.of(convert)); + when(manager.getPluginInfos()).thenReturn(List.of()); + when(convert.getService(ConvertService.class)).thenReturn(new FakeConvert()); + when(source.getService(PluginService.class)).thenReturn(adapter); + return new DatabaseExportServiceImpl(manager, repository); + } + + private static ExportPlan plan(ExportPlan.TablePlan... tables) + { + ExportPlan plan = new ExportPlan(); + plan.setTables(new ArrayList<>(List.of(tables))); + plan.setFormat("FakeConvert"); + plan.setOptions(Map.of("indent", "4")); + plan.setFileName("app export"); + plan.setBatchSize(2); + plan.getStructure().setDrop(true); + plan.getStructure().setIndexes(false); + return plan; + } + + @Test + @SuppressWarnings("unchecked") + public void theCheckFlagsUnknownFieldsAndNamesTheFiles() + { + DatabaseExportService service = service(); + CommonResponse> checked = service.check("demo", "app", plan(table("chat", List.of("id", "missing")), table("logs", List.of()), table("ghost", List.of()))); + assertTrue(checked.getStatus(), String.valueOf(checked.getMessage())); + List> tables = (List>) checked.getData().get("tables"); + assertEquals(List.of("error", "pass", "error"), tables.stream().map(row -> row.get("status")).collect(java.util.stream.Collectors.toList())); + assertEquals(List.of("Field missing does not exist"), tables.get(0).get("messages")); + assertEquals(List.of("chat.fake", "logs.fake", "ghost.fake", "schema.sql"), checked.getData().get("files")); + assertEquals("app_export.zip", checked.getData().get("output")); + assertFalse(service.execute("demo", "app", plan(table("chat", List.of("missing")))).getStatus()); + } + + @Test + public void theSampleIsWrittenByTheChosenFormatWithItsOptions() + { + CommonResponse> sample = service().sample("demo", "app", plan(table("chat", List.of("id", "title"))), "chat", 3); + assertTrue(sample.getStatus(), String.valueOf(sample.getMessage())); + assertEquals("[id, title]|[[1, t1], [2, t2], [3, t3]]|{indent=4}", sample.getData().get("content")); + assertTrue(executed.contains("SELECT `id`, `title` FROM `app`.`chat` LIMIT 3 OFFSET 0")); + } + + @Test + @SuppressWarnings("unchecked") + public void aJobReadsInPagesAndZipsEveryTableWithTheSchema() + throws Exception + { + DatabaseExportService service = service(); + CommonResponse> started = service.execute("demo", "app", plan(table("chat", List.of()), table("logs", List.of()))); + assertTrue(started.getStatus(), String.valueOf(started.getMessage())); + String id = String.valueOf(started.getData().get("jobId")); + Map job = service.job(id).getData(); + for (int attempt = 0; attempt < 100 && "running".equals(job.get("status")); attempt++) { + Thread.sleep(20); + job = service.job(id).getData(); + } + assertEquals("done", job.get("status"), String.valueOf(job.get("message"))); + assertEquals("app_export.zip", job.get("file")); + Map chat = ((List>) job.get("tables")).get(0); + assertEquals(3L, chat.get("written")); + assertTrue(executed.contains("SELECT * FROM `app`.`chat` LIMIT 2 OFFSET 2")); + + TableExportService.ExportFile file = service.file(id).getData(); + assertEquals("app_export.zip", file.getName()); + Map entries = new TreeMap<>(); + try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(file.getContent()))) { + for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) { + entries.put(entry.getName(), new String(zip.readAllBytes(), StandardCharsets.UTF_8)); + } + } + assertEquals(List.of("chat.fake", "logs.fake", "schema.sql"), new ArrayList<>(entries.keySet())); + assertEquals("[id, title]|[[1, t1], [2, t2], [3, t3]]|{indent=4}", entries.get("chat.fake")); + assertEquals("DROP TABLE IF EXISTS `app`.`chat`;\nCREATE TABLE `chat` (\n `id` bigint NOT NULL\n);\n\nDROP TABLE IF EXISTS `app`.`logs`;\nCREATE TABLE `logs` (\n `id` bigint NOT NULL\n);\n", + entries.get("schema.sql")); + assertTrue(service.job(id).getStatus(), "kept for two hours"); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java new file mode 100644 index 0000000000..6e7eda717f --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportPlan.java @@ -0,0 +1,69 @@ +package org.devlive.metaforge.service.exporter; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** What a database export writes: which tables, their fields and rows, the format and the output. */ +@Data +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +public class ExportPlan +{ + private List tables = new ArrayList<>(); + /** all: structure and data; data: data only. */ + private String content = "all"; + private Structure structure = new Structure(); + /** An installed convert plugin, e.g. JsonConvert. */ + private String format; + /** Values of the plugin's declared options. */ + private Map options = new LinkedHashMap<>(); + private String fileName; + /** zip: a file per table packed into one ZIP; merge: one file for every table (only for plugins that merge). */ + private String organize = "zip"; + private boolean compress = true; + private int batchSize = 1000; + private int concurrency = 1; + /** batch: read each table in pages of batchSize; single: one query per table. */ + private String reading = "batch"; + /** stop: end the job at the first failing table; skip: export the others. */ + private String failure = "stop"; + /** Keep the job and its file for two hours after the download. */ + private boolean keepRecord = true; + + @Data + @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) + public static class TablePlan + { + private String name; + /** Fields to export; empty exports every field. */ + private List fields = new ArrayList<>(); + /** Conditions the rows must all meet; empty exports every row. */ + private List filters = new ArrayList<>(); + } + + @Data + public static class Filter + { + private String column; + /** =, !=, >, >=, <, <=, LIKE, NOT LIKE, IS NULL, IS NOT NULL */ + private String operator = "="; + private String value; + } + + @Data + public static class Structure + { + private boolean create = true; + private boolean indexes = true; + private boolean drop; + } + + public boolean includesStructure() + { + return "all".equals(content) && (structure.isCreate() || structure.isDrop()); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java new file mode 100644 index 0000000000..31bd01a147 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/exporter/ExportStatements.java @@ -0,0 +1,115 @@ +package org.devlive.metaforge.service.exporter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** The SQL a database export runs: row selects with fields, filters and pages, and the schema file. */ +public final class ExportStatements +{ + public static final Set OPERATORS = Set.of("=", "!=", ">", ">=", "<", "<=", "LIKE", "NOT LIKE", "IS NULL", "IS NOT NULL"); + private static final Pattern INDEX_LINE = Pattern.compile("^\\s*(UNIQUE |FULLTEXT |SPATIAL )?(KEY|INDEX)\\s|^\\s*CONSTRAINT\\s", Pattern.CASE_INSENSITIVE); + + private final UnaryOperator identifier; + private final UnaryOperator literal; + + public ExportStatements(UnaryOperator identifier, UnaryOperator literal) + { + this.identifier = identifier; + this.literal = literal; + } + + public String target(String database, String table) + { + return identifier.apply(database) + "." + identifier.apply(table); + } + + /** WHERE clause of a table's filters, empty without filters. */ + public String where(List filters) + { + if (filters == null || filters.isEmpty()) { + return ""; + } + List conditions = new ArrayList<>(); + for (ExportPlan.Filter filter : filters) { + String operator = filter.getOperator() == null ? "=" : filter.getOperator().toUpperCase(Locale.ROOT); + if (!OPERATORS.contains(operator)) { + throw new IllegalArgumentException("Unsupported operator " + filter.getOperator()); + } + String column = identifier.apply(filter.getColumn()); + conditions.add(operator.startsWith("IS") ? column + " " + operator : column + " " + operator + " " + literal.apply(filter.getValue() == null ? "" : filter.getValue())); + } + return " WHERE " + String.join(" AND ", conditions); + } + + /** The select of one page (limit > 0) or of every row (limit <= 0). */ + public String select(String database, ExportPlan.TablePlan table, long limit, long offset) + { + String fields = table.getFields() == null || table.getFields().isEmpty() ? "*" + : table.getFields().stream().map(identifier).collect(Collectors.joining(", ")); + String sql = "SELECT " + fields + " FROM " + target(database, table.getName()) + where(table.getFilters()); + return limit > 0 ? sql + " LIMIT " + limit + " OFFSET " + offset : sql; + } + + public String count(String database, ExportPlan.TablePlan table) + { + return "SELECT COUNT(*) FROM " + target(database, table.getName()) + where(table.getFilters()); + } + + /** The schema.sql part of one table: DROP and CREATE as chosen, secondary indexes and constraints left out on request. */ + public String schema(String database, String table, String create, ExportPlan.Structure structure) + { + StringBuilder sql = new StringBuilder(); + if (structure.isDrop()) { + sql.append("DROP TABLE IF EXISTS ").append(target(database, table)).append(";\n"); + } + if (structure.isCreate() && create != null && !create.isEmpty()) { + String statement = structure.isIndexes() ? create : withoutIndexes(create); + sql.append(statement.trim().replaceAll(";+$", "")).append(";\n"); + } + return sql.toString(); + } + + /** CREATE TABLE without KEY, INDEX and CONSTRAINT lines; columns and the primary key stay. */ + static String withoutIndexes(String create) + { + List lines = new ArrayList<>(); + for (String line : create.split("\n")) { + if (!INDEX_LINE.matcher(line).find()) { + lines.add(line); + } + } + // The line before the closing parenthesis must not end with a comma any more. + for (int i = lines.size() - 1; i > 0; i--) { + if (lines.get(i).trim().startsWith(")")) { + String previous = lines.get(i - 1); + lines.set(i - 1, previous.replaceAll(",\\s*$", "")); + break; + } + } + return String.join("\n", lines); + } + + /** The file of a table, with the plugin's extension. */ + public static String fileName(String table, String extension) + { + return table + "." + extension; + } + + /** A name for a file: path separators and reserved characters replaced. */ + public static String safeName(String name, String fallback) + { + String safe = name == null ? "" : name.trim().replaceAll("[\\\\/:*?\"<>|\\s]+", "_"); + return safe.isEmpty() ? fallback : safe; + } + + public static boolean known(Collection names, String name) + { + return name != null && names.contains(name); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java new file mode 100644 index 0000000000..74f584ba2b --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/DatabaseExportService.java @@ -0,0 +1,28 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.exporter.ExportPlan; + +import java.util.List; +import java.util.Map; + +public interface DatabaseExportService +{ + /** Installed export formats with their file extension, version and the settings they declare. */ + CommonResponse>> formats(); + + /** The first rows of one table as read and as the chosen format writes them. */ + CommonResponse> sample(String code, String database, ExportPlan plan, String table, int rows); + + /** Checks every table of a plan and lists the files the export will write. */ + CommonResponse> check(String code, String database, ExportPlan plan); + + /** Starts the export in the background and returns its job id. */ + CommonResponse> execute(String code, String database, ExportPlan plan); + + /** Progress and result of an export job. */ + CommonResponse> job(String id); + + /** The finished file of an export job. */ + CommonResponse file(String id); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java new file mode 100644 index 0000000000..4d06f10387 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/DatabaseExportServiceImpl.java @@ -0,0 +1,621 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.convert.ConvertFilter; +import org.devlive.metaforge.convert.ConvertService; +import org.devlive.metaforge.convert.model.ConvertChoice; +import org.devlive.metaforge.convert.model.ConvertOption; +import org.devlive.metaforge.convert.model.ConvertRequest; +import org.devlive.metaforge.convert.model.ConvertResponse; +import org.devlive.metaforge.plugin.Plugin; +import org.devlive.metaforge.plugin.PluginManager; +import org.devlive.metaforge.plugin.PluginMetadata; +import org.devlive.metaforge.plugin.PluginType; +import org.devlive.metaforge.service.entity.SourceEntity; +import org.devlive.metaforge.service.exporter.ExportPlan; +import org.devlive.metaforge.service.exporter.ExportStatements; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DatabaseExportService; +import org.devlive.metaforge.service.service.TableExportService; +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.Response; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Exports tables of one database: chosen fields and rows of each table through an installed + * convert plugin with its declared settings, plus a schema.sql of their structure, packed into + * one ZIP (or left as the single file) by a background job the page follows and downloads. + */ +@Slf4j +@Service +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2", "REC_CATCH_EXCEPTION"}, justification = "Collaborators are Spring-managed singletons; a table's failure of any kind is reported on the table") +public class DatabaseExportServiceImpl + implements DatabaseExportService +{ + private static final String RAW = "NoneConvert"; + private static final String SCHEMA = "schema.sql"; + private static final long KEEP = Duration.ofHours(2).toMillis(); + + private final PluginManager pluginManager; + private final SourceRepository repository; + private final Map jobs = new ConcurrentHashMap<>(); + private final ExecutorService runner = Executors.newCachedThreadPool(task -> { + Thread thread = new Thread(task, "metaforge-export"); + thread.setDaemon(true); + return thread; + }); + + public DatabaseExportServiceImpl(PluginManager pluginManager, SourceRepository repository) + { + this.pluginManager = pluginManager; + this.repository = repository; + } + + /** Jobs belong to the user who started them. */ + private static String currentUser() + { + return String.valueOf(UserDetailsService.getUser().getCode()); + } + + // ---- formats ---- + + @Override + public CommonResponse>> formats() + { + List> formats = new ArrayList<>(); + for (PluginMetadata plugin : pluginManager.getPluginInfos()) { + if (plugin.getType() != PluginType.CONVERT || RAW.equals(plugin.getName())) { + continue; + } + Optional converter = ConvertFilter.filter(pluginManager, plugin.getName()); + if (converter.isEmpty()) { + continue; + } + Map format = new LinkedHashMap<>(); + format.put("name", plugin.getName()); + format.put("version", plugin.getVersion()); + format.put("extension", converter.get().extension()); + format.put("merge", false); + format.put("options", converter.get().options().stream().map(DatabaseExportServiceImpl::option).collect(Collectors.toList())); + formats.add(format); + } + return CommonResponse.success(formats); + } + + private static Map option(ConvertOption option) + { + Map value = new LinkedHashMap<>(); + value.put("name", option.getName()); + value.put("labels", option.getLabels()); + value.put("defaultValue", option.getDefaultValue()); + List> choices = new ArrayList<>(); + for (ConvertChoice choice : option.getChoices()) { + Map item = new LinkedHashMap<>(); + item.put("value", choice.getValue()); + item.put("labels", choice.getLabels()); + choices.add(item); + } + value.put("choices", choices); + return value; + } + + // ---- shared ---- + + /** A source of the export: its plugin service, a raw-row configure and what its database holds. */ + private final class Target + { + private PluginService service; + private SourceEntity entity; + private Plugin plugin; + private ExportStatements statements; + /** table -> column -> type, in column order. */ + private Map> columns; + + RuntimeConfigure raw() + { + return entity.toConfigure(RAW, pluginManager, plugin); + } + } + + private CommonResponse target(String code, String database, ExportPlan plan) + { + Optional entity = repository.findByCode(code); + if (entity.isEmpty()) { + return CommonResponse.failure(String.format("Resource [ %s ] not found", code)); + } + Optional plugin = pluginManager.getPlugin(entity.get().getType()); + if (plugin.isEmpty()) { + return CommonResponse.failure(String.format("Plugin [ %s ] not found", entity.get().getType())); + } + Target target = new Target(); + target.entity = entity.get(); + target.plugin = plugin.get(); + target.service = plugin.get().getService(PluginService.class); + if (!target.service.managementCapabilities().supports("table:export")) { + return CommonResponse.failure("Source does not support table:export"); + } + if (plan != null && plan.includesStructure() && !target.service.managementCapabilities().supports("table:statement")) { + return CommonResponse.failure("Source does not support table:statement"); + } + target.statements = new ExportStatements(target.service::sqlIdentifier, target.service::sqlLiteral); + Response response = target.service.execute(target.raw(), "SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = " + + target.service.sqlLiteral(database) + " ORDER BY TABLE_NAME, ORDINAL_POSITION"); + if (response == null || !Boolean.TRUE.equals(response.getIsSuccessful())) { + return CommonResponse.failure(response == null ? "Reading the columns failed" : response.getMessage()); + } + target.columns = new LinkedHashMap<>(); + for (Object row : response.getColumns()) { + List values = values(row, 3); + target.columns.computeIfAbsent(values.get(0), table -> new LinkedHashMap<>()).put(values.get(1), values.get(2).toUpperCase(Locale.ROOT)); + } + return CommonResponse.success(target); + } + + private static List values(Object row, int count) + { + List values = new ArrayList<>(); + if (row instanceof List) { + ((List) row).forEach(value -> values.add(value == null ? null : String.valueOf(value))); + } + else if (row instanceof JsonNode) { + ((JsonNode) row).elements().forEachRemaining(value -> values.add(value.isNull() ? null : value.asText())); + } + while (values.size() < count) { + values.add(""); + } + return values; + } + + private Optional converter(ExportPlan plan) + { + String format = plan.getFormat() == null ? "" : plan.getFormat(); + return format.isEmpty() || RAW.equals(format) ? Optional.empty() : ConvertFilter.filter(pluginManager, format); + } + + /** Problems of one table in the plan, empty when it can be exported. */ + private static List problems(Target target, ExportPlan.TablePlan table) + { + List problems = new ArrayList<>(); + Map columns = target.columns.get(table.getName()); + if (columns == null) { + problems.add("Table " + table.getName() + " does not exist"); + return problems; + } + for (String field : table.getFields()) { + if (!columns.containsKey(field)) { + problems.add("Field " + field + " does not exist"); + } + } + for (ExportPlan.Filter filter : table.getFilters()) { + String operator = filter.getOperator() == null ? "" : filter.getOperator().toUpperCase(Locale.ROOT); + if (!columns.containsKey(filter.getColumn())) { + problems.add("Filter field " + filter.getColumn() + " does not exist"); + } + else if (!ExportStatements.OPERATORS.contains(operator)) { + problems.add("Unsupported operator " + filter.getOperator()); + } + else if (!operator.startsWith("IS") && (filter.getValue() == null || filter.getValue().isEmpty())) { + problems.add("Filter on " + filter.getColumn() + " needs a value"); + } + } + return problems; + } + + private static String rule(ExportPlan.TablePlan table) + { + return table.getFilters().isEmpty() ? "all" : "filtered"; + } + + // ---- sample ---- + + @Override + public CommonResponse> sample(String code, String database, ExportPlan plan, String table, int rows) + { + Optional converter = converter(plan); + CommonResponse resolved = target(code, database, null); + if (!Boolean.TRUE.equals(resolved.getStatus())) { + return CommonResponse.failure(String.valueOf(resolved.getMessage())); + } + Target target = resolved.getData(); + ExportPlan.TablePlan chosen = plan.getTables().stream().filter(item -> table.equals(item.getName())).findFirst().orElseGet(() -> { + ExportPlan.TablePlan item = new ExportPlan.TablePlan(); + item.setName(table); + return item; + }); + List problems = problems(target, chosen); + if (!problems.isEmpty()) { + return CommonResponse.failure(problems.get(0)); + } + Response response = target.service.execute(target.raw(), target.statements.select(database, chosen, Math.max(1, Math.min(rows, 20)), 0)); + if (response == null || !Boolean.TRUE.equals(response.getIsSuccessful())) { + return CommonResponse.failure(response == null ? "Reading the rows failed" : response.getMessage()); + } + Map result = new LinkedHashMap<>(); + result.put("headers", response.getHeaders()); + result.put("rows", response.getColumns()); + if (converter.isPresent()) { + File directory = null; + try { + directory = Files.createTempDirectory("metaforge-export-sample").toFile(); + File file = write(converter.get(), plan, target, database, chosen, response, directory); + result.put("file", file.getName()); + result.put("content", FileUtils.readFileToString(file, StandardCharsets.UTF_8)); + } + catch (IOException | RuntimeException error) { + return CommonResponse.failure(error.getMessage() == null ? "The format could not write the sample" : error.getMessage()); + } + finally { + FileUtils.deleteQuietly(directory); + } + } + return CommonResponse.success(result); + } + + /** Writes rows through the plugin into the directory and returns the file it wrote. */ + private static File write(ConvertService converter, ExportPlan plan, Target target, String database, ExportPlan.TablePlan table, Response rows, File directory) + throws IOException + { + Map types = target.columns.getOrDefault(table.getName(), Map.of()); + ConvertRequest request = new ConvertRequest(); + request.setName(table.getName()); + request.setPath(directory.getAbsolutePath()); + request.setDelimiter("\t"); + request.setHeaders(new ArrayList<>(rows.getHeaders())); + request.setColumns(rows.getColumns()); + request.setTypes(rows.getHeaders().stream().map(header -> types.getOrDefault(header, "")).collect(Collectors.toList())); + request.setOptions(plan.getOptions() == null ? Map.of() : plan.getOptions()); + ConvertResponse written = converter.writer(request); + if (!Boolean.TRUE.equals(written.getSuccessful())) { + throw new IOException(written.getMessage() == null ? "The format could not write " + table.getName() : written.getMessage()); + } + File[] files = directory.listFiles(file -> file.getName().startsWith(table.getName() + ".")); + if (files == null || files.length != 1) { + throw new IOException("The format wrote no file for " + table.getName()); + } + return files[0]; + } + + // ---- check ---- + + @Override + public CommonResponse> check(String code, String database, ExportPlan plan) + { + Optional converter = converter(plan); + if (converter.isEmpty()) { + return CommonResponse.failure("Choose an installed export format"); + } + if (plan.getTables().isEmpty()) { + return CommonResponse.failure("Choose at least one table"); + } + if ("merge".equals(plan.getOrganize())) { + return CommonResponse.failure("The format " + plan.getFormat() + " can not merge tables into one file"); + } + CommonResponse resolved = target(code, database, plan); + if (!Boolean.TRUE.equals(resolved.getStatus())) { + return CommonResponse.failure(String.valueOf(resolved.getMessage())); + } + Target target = resolved.getData(); + String extension = converter.get().extension(); + List> tables = new ArrayList<>(); + int errors = 0; + for (ExportPlan.TablePlan table : plan.getTables()) { + List problems = problems(target, table); + errors += problems.isEmpty() ? 0 : 1; + Map row = new LinkedHashMap<>(); + row.put("name", table.getName()); + row.put("structure", plan.includesStructure()); + row.put("data", true); + row.put("rule", rule(table)); + row.put("file", ExportStatements.fileName(table.getName(), extension)); + row.put("status", problems.isEmpty() ? "pass" : "error"); + row.put("messages", problems); + tables.add(row); + } + List files = tables.stream().map(row -> String.valueOf(row.get("file"))).collect(Collectors.toList()); + if (plan.includesStructure()) { + files.add(SCHEMA); + } + Map result = new LinkedHashMap<>(); + result.put("tables", tables); + result.put("files", files); + result.put("output", output(plan, files.size(), extension, plan.getTables().get(0).getName())); + result.put("errors", errors); + return CommonResponse.success(result); + } + + /** One file without compression is downloaded as it is; otherwise everything goes into a ZIP. */ + private static String output(ExportPlan plan, int files, String extension, String table) + { + String base = ExportStatements.safeName(plan.getFileName(), "export"); + return files == 1 && !plan.isCompress() ? ExportStatements.fileName(ExportStatements.safeName(table, base), extension) : base + ".zip"; + } + + // ---- execute ---- + + /** A running or finished export; tables keep their progress, the file is written when all are done. */ + private static final class Job + { + private final String id = UUID.randomUUID().toString(); + private final String user; + private final long created = System.currentTimeMillis(); + private final boolean keep; + private final Map> tables = new LinkedHashMap<>(); + private final AtomicBoolean stopped = new AtomicBoolean(); + private volatile String status = "running"; + private volatile String message; + private volatile File file; + private volatile String name; + private volatile File directory; + + private Job(String user, boolean keep) + { + this.user = user; + this.keep = keep; + } + } + + @Override + public CommonResponse> execute(String code, String database, ExportPlan plan) + { + CommonResponse> checked = check(code, database, plan); + if (!Boolean.TRUE.equals(checked.getStatus())) { + return checked; + } + if (((Number) checked.getData().get("errors")).intValue() > 0) { + return CommonResponse.failure("Fix the tables that do not pass the check first"); + } + ConvertService converter = converter(plan).orElseThrow(); + Target target = target(code, database, plan).getData(); + expire(); + Job job = new Job(currentUser(), plan.isKeepRecord()); + for (ExportPlan.TablePlan table : plan.getTables()) { + Map item = new ConcurrentHashMap<>(); + item.put("name", table.getName()); + item.put("status", "waiting"); + item.put("written", 0L); + item.put("total", -1L); + job.tables.put(table.getName(), item); + } + jobs.put(job.id, job); + String output = String.valueOf(checked.getData().get("output")); + runner.submit(() -> run(job, target, converter, database, plan, output)); + Map result = new LinkedHashMap<>(); + result.put("jobId", job.id); + return CommonResponse.success(result); + } + + private void run(Job job, Target target, ConvertService converter, String database, ExportPlan plan, String output) + { + Map schemas = new ConcurrentHashMap<>(); + ExecutorService pool = Executors.newFixedThreadPool(Math.max(1, Math.min(4, plan.getConcurrency()))); + try { + job.directory = Files.createTempDirectory("metaforge-export").toFile(); + for (ExportPlan.TablePlan table : plan.getTables()) { + pool.submit(() -> exportTable(job, target, converter, database, plan, table, schemas)); + } + pool.shutdown(); + if (!pool.awaitTermination(6, TimeUnit.HOURS)) { + pool.shutdownNow(); + job.stopped.set(true); + job.message = "The export took longer than six hours"; + } + if (job.stopped.get()) { + job.status = "failed"; + job.message = job.message == null ? "The export stopped at a failing table" : job.message; + return; + } + if (plan.includesStructure()) { + String schema = plan.getTables().stream().map(table -> schemas.getOrDefault(table.getName(), "")).collect(Collectors.joining("\n")); + FileUtils.writeStringToFile(new File(job.directory, SCHEMA), schema, StandardCharsets.UTF_8); + } + job.file = pack(job.directory, plan, output); + job.name = job.file.getParentFile().equals(job.directory) ? job.file.getName() : output; + boolean failed = job.tables.values().stream().anyMatch(item -> "failed".equals(item.get("status"))); + job.status = failed ? "partial" : "done"; + } + catch (IOException | RuntimeException error) { + log.warn("Export {} failed", job.id, error); + job.status = "failed"; + job.message = error.getMessage(); + } + catch (InterruptedException error) { + Thread.currentThread().interrupt(); + job.status = "failed"; + job.message = "The export was interrupted"; + } + finally { + pool.shutdownNow(); + } + } + + private void exportTable(Job job, Target target, ConvertService converter, String database, ExportPlan plan, ExportPlan.TablePlan table, Map schemas) + { + Map item = job.tables.get(table.getName()); + if (job.stopped.get()) { + item.put("status", "cancelled"); + return; + } + item.put("status", "running"); + RuntimeConfigure configure = target.raw(); + try { + if (plan.includesStructure()) { + Response create = target.service.execute(configure, "SHOW CREATE TABLE " + target.statements.target(database, table.getName())); + if (create == null || !Boolean.TRUE.equals(create.getIsSuccessful()) || create.getColumns().isEmpty()) { + throw new IOException(create == null ? "Reading the structure failed" : create.getMessage()); + } + String statement = values(create.getColumns().get(0), 2).get(1); + schemas.put(table.getName(), target.statements.schema(database, table.getName(), statement, plan.getStructure())); + } + Response total = target.service.execute(configure, target.statements.count(database, table)); + if (total != null && Boolean.TRUE.equals(total.getIsSuccessful()) && !total.getColumns().isEmpty()) { + item.put("total", parseLong(values(total.getColumns().get(0), 1).get(0))); + } + Response rows = read(job, target, configure, database, plan, table, item); + write(converter, plan, target, database, table, rows, job.directory); + item.put("status", "done"); + } + catch (IOException | RuntimeException error) { + log.warn("Export of {}.{} failed", database, table.getName(), error); + item.put("status", "failed"); + item.put("message", error.getMessage() == null ? "Export failed" : error.getMessage()); + if ("stop".equals(plan.getFailure())) { + job.stopped.set(true); + job.message = table.getName() + ": " + item.get("message"); + } + } + } + + /** Every row of a table, read in pages of batchSize (or in one query), with the written count kept current. */ + private static Response read(Job job, Target target, RuntimeConfigure configure, String database, ExportPlan plan, ExportPlan.TablePlan table, Map item) + throws IOException + { + long batch = "single".equals(plan.getReading()) ? 0 : Math.max(1, Math.min(100_000, plan.getBatchSize())); + Response all = null; + List rows = new ArrayList<>(); + for (long offset = 0; ; offset += batch) { + if (job.stopped.get()) { + throw new IOException("Stopped"); + } + Response page = target.service.execute(configure, target.statements.select(database, table, batch, offset)); + if (page == null || !Boolean.TRUE.equals(page.getIsSuccessful())) { + throw new IOException(page == null ? "Reading the rows failed" : page.getMessage()); + } + all = all == null ? page : all; + rows.addAll(page.getColumns()); + item.put("written", (long) rows.size()); + if (batch == 0 || page.getColumns().size() < batch) { + break; + } + } + all.setColumns(rows); + return all; + } + + private static File pack(File directory, ExportPlan plan, String output) + throws IOException + { + File[] files = directory.listFiles(); + if (files == null || files.length == 0) { + throw new IOException("The export wrote no file"); + } + Arrays.sort(files); + if (!output.endsWith(".zip") && files.length == 1) { + return files[0]; + } + File zip = new File(directory.getParentFile(), directory.getName() + ".zip"); + try (OutputStream out = Files.newOutputStream(zip.toPath()); ZipOutputStream stream = new ZipOutputStream(out)) { + stream.setLevel(plan.isCompress() ? Deflater.DEFAULT_COMPRESSION : Deflater.NO_COMPRESSION); + for (File file : files) { + stream.putNextEntry(new ZipEntry(file.getName())); + Files.copy(file.toPath(), stream); + stream.closeEntry(); + } + } + return zip; + } + + private static long parseLong(String value) + { + if (value == null) { + return -1; + } + try { + return Long.parseLong(value.trim()); + } + catch (NumberFormatException error) { + return -1; + } + } + + // ---- job ---- + + private Job owned(String id) + { + Job job = jobs.get(id); + return job == null || !job.user.equals(currentUser()) ? null : job; + } + + @Override + public CommonResponse> job(String id) + { + Job job = owned(id); + if (job == null) { + return CommonResponse.failure("Export job not found"); + } + Map result = new LinkedHashMap<>(); + result.put("status", job.status); + result.put("message", job.message); + result.put("tables", new ArrayList<>(job.tables.values())); + File file = job.file; + if (file != null) { + result.put("file", job.name); + result.put("size", file.length()); + } + return CommonResponse.success(result); + } + + @Override + public CommonResponse file(String id) + { + Job job = owned(id); + if (job == null || job.file == null) { + return CommonResponse.failure("The export file is not ready"); + } + try { + TableExportService.ExportFile file = new TableExportService.ExportFile(job.name, FileUtils.readFileToByteArray(job.file)); + if (!job.keep) { + remove(job); + } + return CommonResponse.success(file); + } + catch (IOException error) { + return CommonResponse.failure(error.getMessage()); + } + } + + /** Drops finished jobs older than two hours with their files. */ + private void expire() + { + long now = System.currentTimeMillis(); + jobs.values().stream().filter(job -> !"running".equals(job.status) && now - job.created > KEEP).collect(Collectors.toList()).forEach(this::remove); + } + + private void remove(Job job) + { + jobs.remove(job.id); + if (job.file != null && job.directory != null && !job.file.getParentFile().equals(job.directory)) { + FileUtils.deleteQuietly(job.file); + } + FileUtils.deleteQuietly(job.directory); + } +} From b738cb0eae1e6c8ce4f717cb43f196c0e7bb2a9e Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:27:46 -0400 Subject: [PATCH 006/232] refactor(spi): drop the unused isSupportMeta flag Nothing read the flag since plugins declare their capabilities, so the SPI default, the four overrides and the source list field go away. --- .../devlive/metaforge/service/entity/SourceEntity.java | 4 ---- .../service/service/impl/SourceServiceImpl.java | 2 -- .../java/org/devlive/metaforge/spi/PluginService.java | 10 ---------- .../devlive/metaforge/plugin/ClickHouseService.java | 6 ------ .../org/devlive/metaforge/plugin/DorisService.kt | 5 ----- .../org/devlive/metaforge/plugin/MySQLService.java | 6 ------ .../org/devlive/metaforge/plugin/StarRocksService.kt | 5 ----- 7 files changed, 38 deletions(-) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java index cea59972f0..f850b82cbb 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/SourceEntity.java @@ -137,10 +137,6 @@ public class SourceEntity @JsonProperty(value = "configure") private PluginConfigure configure; - @Transient - @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) - private Boolean isSupportMeta; - public void setOriginalConfigure(String configure) { this.originalConfigure = configure; diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java index d8448351c8..6e1f4534f1 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SourceServiceImpl.java @@ -92,8 +92,6 @@ public CommonResponse> getAll(BaseRepository Date: Sat, 31 Oct 2026 22:27:46 -0400 Subject: [PATCH 007/232] fix(web): title sign-in, sign-up and other untitled pages Sign-in, sign-up, setup, home, 404 and 403 had no route title, so the browser tab only showed MetaForge. They now show their page name, and the title is rewritten when the language changes. --- core/metaforge-web/src/i18n/locales/en.json | 2 ++ core/metaforge-web/src/i18n/locales/zh-cn.json | 2 ++ core/metaforge-web/src/router/auth.ts | 2 ++ core/metaforge-web/src/router/default.ts | 2 +- core/metaforge-web/src/router/http.ts | 2 ++ core/metaforge-web/src/router/index.ts | 11 +++++++---- core/metaforge-web/src/router/setup.ts | 1 + 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/core/metaforge-web/src/i18n/locales/en.json b/core/metaforge-web/src/i18n/locales/en.json index b201d40495..44312d53db 100644 --- a/core/metaforge-web/src/i18n/locales/en.json +++ b/core/metaforge-web/src/i18n/locales/en.json @@ -1271,6 +1271,8 @@ "navigation.dataExport": "Data export", "navigation.dataImport": "Data import", "navigation.erDiagram": "ER diagram", + "navigation.pageNotAuthorized": "Not authorized", + "navigation.pageNotFound": "Page not found", "navigation.systemManagement": "Administration", "notify.common.apply": "Apply", "notify.common.batch": "Batch Settings", diff --git a/core/metaforge-web/src/i18n/locales/zh-cn.json b/core/metaforge-web/src/i18n/locales/zh-cn.json index 015ed7a685..8e247883cc 100644 --- a/core/metaforge-web/src/i18n/locales/zh-cn.json +++ b/core/metaforge-web/src/i18n/locales/zh-cn.json @@ -1271,6 +1271,8 @@ "navigation.dataExport": "数据导出", "navigation.dataImport": "数据导入", "navigation.erDiagram": "ER 图", + "navigation.pageNotAuthorized": "无权访问", + "navigation.pageNotFound": "页面未找到", "navigation.systemManagement": "系统管理", "notify.common.apply": "应用", "notify.common.batch": "批量设置", diff --git a/core/metaforge-web/src/router/auth.ts b/core/metaforge-web/src/router/auth.ts index 83298e2401..56a680fdef 100644 --- a/core/metaforge-web/src/router/auth.ts +++ b/core/metaforge-web/src/router/auth.ts @@ -8,11 +8,13 @@ const createAuthRoute = (router: Router) => { { name: 'signin', path: 'signin', + meta: { title: 'user.common.signin' }, component: () => import('@/views/auth/signin/AuthSignin.vue') }, { name: 'signup', path: 'signup', + meta: { title: 'user.common.signup' }, component: () => import('@/views/auth/signup/AuthSignup.vue') } ] diff --git a/core/metaforge-web/src/router/default.ts b/core/metaforge-web/src/router/default.ts index ada7e4ea0b..b6f538d1a3 100644 --- a/core/metaforge-web/src/router/default.ts +++ b/core/metaforge-web/src/router/default.ts @@ -12,7 +12,7 @@ const createDefaultRouter = (router: Router) => { component: LayoutContainer, meta: { requiresAuth: true }, children: [ - { name: 'home', path: 'home', component: () => import('@/views/pages/default/dashboard/DashboardHome.vue') }, + { name: 'home', path: 'home', meta: { title: 'common.home' }, component: () => import('@/views/pages/default/dashboard/DashboardHome.vue') }, { name: 'store', path: 'store', meta: { title: 'common.store' }, component: () => import('@/views/pages/store/StoreHome.vue') }, { name: 'workspace', path: 'workspace', redirect: '/query' }, { name: 'analysis', path: 'analysis', redirect: '/dashboard' }, diff --git a/core/metaforge-web/src/router/http.ts b/core/metaforge-web/src/router/http.ts index c4b7ae6e49..59faddc682 100644 --- a/core/metaforge-web/src/router/http.ts +++ b/core/metaforge-web/src/router/http.ts @@ -8,11 +8,13 @@ const createHttpRoute = (router: Router) => { { name: '404', path: '404', + meta: { title: 'navigation.pageNotFound' }, component: () => import('@/views/common/error/NotFound.vue') }, { name: '403', path: '403', + meta: { title: 'navigation.pageNotAuthorized' }, component: () => import('@/views/common/error/NotAuthorized.vue') }, { diff --git a/core/metaforge-web/src/router/index.ts b/core/metaforge-web/src/router/index.ts index 212f90ad77..dbbce73bcc 100644 --- a/core/metaforge-web/src/router/index.ts +++ b/core/metaforge-web/src/router/index.ts @@ -1,3 +1,4 @@ +import { watch } from 'vue' import { createRouter, createWebHashHistory, RouteRecordRaw } from 'vue-router' import i18n from '@/i18n/I18n' import { createAuthRoute } from '@/router/auth' @@ -54,12 +55,14 @@ export function invalidateSetupCheck(): void setupChecked = undefined } -// 标题统一为「页面名 - MetaForge」；无页面标题的路由只显示 MetaForge。 -router.afterEach(to => { - const key = to.meta?.title +// 标题统一为「页面名 - MetaForge」；无页面标题的路由只显示 MetaForge。切换语言后按新语言重写。 +function refreshTitle() { + const key = router.currentRoute.value.meta?.title const page = typeof key === 'string' ? i18n.global.t(key) : '' document.title = page && page !== key ? `${page} - MetaForge` : 'MetaForge' -}) +} +router.afterEach(refreshTitle) +watch(i18n.global.locale, refreshTitle) router.beforeEach(async (_to, _from, _next) => { // 404 的"返回"回到跳转前所在的页面；不存在的地址本身回跳只会再次 404。 diff --git a/core/metaforge-web/src/router/setup.ts b/core/metaforge-web/src/router/setup.ts index b530c7828b..0a769feb4d 100644 --- a/core/metaforge-web/src/router/setup.ts +++ b/core/metaforge-web/src/router/setup.ts @@ -8,6 +8,7 @@ const createSetupRoute = (router: Router) => { { name: 'setup_home', path: '', + meta: { title: 'common.install' }, component: () => import('@/views/setup/SetupHome.vue') } ] From 09e6ae1bd686d3140c76d593a1a5c7ee3a60cb84 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:30:16 -0400 Subject: [PATCH 008/232] fix(source): skip configure fields sent without a value Only blank strings counted as empty, so a custom configures array sent as null reached getMapValue and testing the connection failed with an NPE. Any field without a value is now left out, and a null array gives no environment. --- .../service/common/ConfigureUtils.java | 7 +++++ .../service/common/ConfigureUtilsTest.java | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java index 2c65dab909..6589370640 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/common/ConfigureUtils.java @@ -187,8 +187,12 @@ public static List filterNotEmpty(List !isEmpty(v)).collect(Collectors.toList()); } + /** A field without a value is empty whatever its type, so it is not applied; a blank string is empty too. */ public static boolean isEmpty(PluginConfigureField field) { + if (field.getValue() == null) { + return true; + } boolean flag = true; switch (field.getType()) { case String: @@ -234,6 +238,9 @@ public static Map getMapValue(List configu PluginConfigureField configureField = getConfigure(configures, key); if (configureField.getType().equals(PluginConfigureFieldType.Array)) { List> list = (List>) configureField.getValue(); + if (list == null) { + return values; + } list.forEach(map -> values.put(String.valueOf(map.get(PluginConfigureFieldName.field.name())), map.get(PluginConfigureFieldName.value.name()))); } else { diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java index 14924f7f64..ff31cf1d7a 100644 --- a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/common/ConfigureUtilsTest.java @@ -2,11 +2,16 @@ import org.devlive.metaforge.plugin.configure.PluginConfigure; import org.devlive.metaforge.plugin.configure.PluginConfigureExecutor; +import org.devlive.metaforge.plugin.configure.PluginConfigureField; +import org.devlive.metaforge.plugin.configure.PluginConfigureFieldName; +import org.devlive.metaforge.plugin.configure.PluginConfigureFieldType; +import org.devlive.metaforge.spi.model.RuntimeConfigure; import org.devlive.metaforge.plugin.configure.PluginConfigurePipelineType; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class ConfigureUtilsTest @@ -61,4 +66,28 @@ public void testFindPipelineExecutorWithNullArguments() Assert.assertNull(ConfigureUtils.findPipelineExecutor(configure, null, null)); Assert.assertSame(matched, ConfigureUtils.findPipelineExecutor(configure, "TRUNCATE", PluginConfigurePipelineType.INPUT)); } + + @Test + public void testFieldsSentWithoutValueAreNotApplied() + { + // Testing a connection whose custom configures were sent as null must not fail with an NPE. + List fields = Arrays.asList( + PluginConfigureField.builder().field(PluginConfigureFieldName.host).type(PluginConfigureFieldType.String).value("127.0.0.1").build(), + PluginConfigureField.builder().field(PluginConfigureFieldName.port).type(PluginConfigureFieldType.Number).build(), + PluginConfigureField.builder().field(PluginConfigureFieldName.configures).type(PluginConfigureFieldType.Array).build()); + + List applied = ConfigureUtils.filterNotEmpty(fields); + Assert.assertEquals(1, applied.size()); + RuntimeConfigure configure = ConfigureUtils.preparedConfigure(applied); + Assert.assertEquals("127.0.0.1", configure.getHost()); + } + + @Test + public void testNullArrayConfiguresGiveNoEnvironment() + { + List fields = Arrays.asList( + PluginConfigureField.builder().field(PluginConfigureFieldName.configures).type(PluginConfigureFieldType.Array).build()); + + Assert.assertTrue(ConfigureUtils.getMapValue(fields, PluginConfigureFieldName.configures).isEmpty()); + } } From 7ea5fbc72c4da198a6bfc476d55a9d67ff63940e Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:37:02 -0400 Subject: [PATCH 009/232] fix(web): return to sign-in, sign-up and setup after reconnecting Entry pages were never kept as a return address, so a network drop on sign-in, sign-up or setup sent the user to home after reconnecting and sign-up was lost. The network page now keeps them with their own redirect; signing in and the other interim pages still never return to an entry page. --- .../src/composables/useRouteRedirect.ts | 34 ++++++++++++------- .../src/views/common/error/NotNetwork.vue | 3 +- .../tests/route-redirect.test.mjs | 15 ++++++++ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/core/metaforge-web/src/composables/useRouteRedirect.ts b/core/metaforge-web/src/composables/useRouteRedirect.ts index ae08413c0a..77255db5b8 100644 --- a/core/metaforge-web/src/composables/useRouteRedirect.ts +++ b/core/metaforge-web/src/composables/useRouteRedirect.ts @@ -13,36 +13,46 @@ type LocationLike = Pick // 中间页自身不能作为回跳目标，否则恢复时会在中间页之间循环。 const INTERIM_PATH = /^\/(?:common\/(?:404|403|not_network)|auth|setup)(?:[/?#]|$)/i +// 登录、注册、安装是用户自己停留的页面：断网恢复后要回到它们本身（连同它们携带的 redirect）， +// 只有断网页记录它们；登录成功等其他回跳仍不会落到这些页面。 +const ENTRY_PATH = /^\/(?:auth|setup)(?:[/?#]|$)/i +const NETWORK_PATH = '/common/not_network' + export function isInterimPath(path: string): boolean { return INTERIM_PATH.test(path) } /** 只接受站内相对路径（`/` 开头、非 `//`、无反斜杠与控制字符），并排除中间页，防止开放重定向。 */ -export function resolveRedirect(value: unknown, fallback = '/home'): string { +export function resolveRedirect(value: unknown, fallback = '/home', entry = false): string { const candidate = Array.isArray(value) ? value[0] : value if (typeof candidate !== 'string' || !candidate.startsWith('/') || candidate.startsWith('//') || candidate.includes('\\') || /[\u0000-\u001f\u007f]/.test(candidate)) { return fallback } const target = canonicalPagePath(candidate) - return isInterimPath(target) ? fallback : target + return isInterimPath(target) && !(entry && ENTRY_PATH.test(target)) ? fallback : target } -/** 被打断的地址：普通页面取自身 fullPath；已在中间页时沿用它携带的 redirect，保证多次跳转不丢原地址。 */ -export function interruptedTarget(location: LocationLike | undefined): string { +/** + * 被打断的地址：普通页面取自身 fullPath；已在中间页时沿用它携带的 redirect，保证多次跳转不丢原地址。 + * `entry` 为 true（断网）时登录、注册、安装页取自身 fullPath。 + */ +export function interruptedTarget(location: LocationLike | undefined, entry = false): string { if (!location) return '' - if (isInterimPath(location.path)) return resolveRedirect(location.query?.redirect, '') + if (entry && ENTRY_PATH.test(location.path)) return resolveRedirect(location.fullPath, '', true) + if (isInterimPath(location.path)) return resolveRedirect(location.query?.redirect, '', entry) return resolveRedirect(location.fullPath, '') } -export function redirectQuery(target: string): LocationQuery { - const redirect = resolveRedirect(target, '') +export function redirectQuery(target: string, entry = false): LocationQuery { + const redirect = resolveRedirect(target, '', entry) return redirect ? { redirect } : {} } /** 构造前往中间页的地址，原地址作为 `redirect` 查询参数带过去。 */ export function interimLocation(path: string, from: LocationLike | undefined) { - return { path, query: redirectQuery(interruptedTarget(from)) } + const entry = path === NETWORK_PATH + return { path, query: redirectQuery(interruptedTarget(from, entry), entry) } } // 首次加载时请求在守卫里失败，currentRoute 仍是起始位置，需要记下正在进行的导航才能拿到深链接。 @@ -65,14 +75,14 @@ export function currentLocation(router: Router): LocationLike { return pendingLocation ?? router.currentRoute.value } -/** 中间页使用：读取并校验 `redirect`，恢复时 replace 回原地址，没有原地址时回首页。 */ -export function useRouteRedirect(fallback = '/home') { +/** 中间页使用：读取并校验 `redirect`，恢复时 replace 回原地址，没有原地址时回首页。`entry` 见 interruptedTarget。 */ +export function useRouteRedirect(fallback = '/home', entry = false) { const route = useRoute() const router = useRouter() - const original = computed(() => resolveRedirect(route.query.redirect, '')) + const original = computed(() => resolveRedirect(route.query.redirect, '', entry)) const target = computed(() => original.value || fallback) - const query = computed(() => redirectQuery(original.value)) + const query = computed(() => redirectQuery(original.value, entry)) const restore = () => router.replace(target.value) diff --git a/core/metaforge-web/src/views/common/error/NotNetwork.vue b/core/metaforge-web/src/views/common/error/NotNetwork.vue index a9021e0137..b1164052da 100644 --- a/core/metaforge-web/src/views/common/error/NotNetwork.vue +++ b/core/metaforge-web/src/views/common/error/NotNetwork.vue @@ -56,7 +56,8 @@ import { useRouteRedirect } from '@/composables/useRouteRedirect' defineOptions({ name: 'NotNetwork' }) const { t } = useI18n() -const { restore } = useRouteRedirect() +// 断网前停留在登录、注册或安装页时也回到原页。 +const { restore } = useRouteRedirect('/home', true) const checking = ref(false) diff --git a/core/metaforge-web/tests/route-redirect.test.mjs b/core/metaforge-web/tests/route-redirect.test.mjs index 84c9f388a3..e81ad5d497 100644 --- a/core/metaforge-web/tests/route-redirect.test.mjs +++ b/core/metaforge-web/tests/route-redirect.test.mjs @@ -42,6 +42,21 @@ test('interrupted pages hand their address on, interim pages pass along the one assert.deepEqual(redirectQuery('/auth/signin'), {}) }) +test('the network page records sign-in, sign-up and setup themselves; other interim pages still do not', () => { + const signin = { path: '/auth/signin', fullPath: `/auth/signin?redirect=${ encodeURIComponent(workbench) }`, query: { redirect: workbench } } + assert.deepEqual(interimLocation('/common/not_network', signin), { path: '/common/not_network', query: { redirect: signin.fullPath } }) + for (const path of ['/auth/signup', '/setup']) { + assert.deepEqual(interimLocation('/common/not_network', { path, fullPath: path, query: {} }), { path: '/common/not_network', query: { redirect: path } }) + } + // 404 and 403 still pass on the page before them, and the network page never records itself. + assert.deepEqual(interimLocation('/common/not_network', { path: '/common/404', fullPath: '/common/404', query: { redirect: workbench } }), { path: '/common/not_network', query: { redirect: workbench } }) + assert.deepEqual(interimLocation('/common/not_network', { path: '/common/not_network', fullPath: '/common/not_network?redirect=/setup', query: { redirect: '/setup' } }), { path: '/common/not_network', query: { redirect: '/setup' } }) + assert.equal(resolveRedirect('/common/not_network', '', true), '') + // Signing in or other interim pages never go back to an entry page. + assert.deepEqual(interimLocation('/auth/signin', { path: '/auth/signup', fullPath: '/auth/signup', query: {} }), { path: '/auth/signin', query: {} }) + assert.equal(resolveRedirect('/auth/signup'), '/home') +}) + test('the pending deep link is reported while the first navigation is still in its guards', async () => { const component = { render: () => null } const router = createRouter({ history: createMemoryHistory(), routes: ['/home', '/manage', '/common/404', '/common/not_network'].map(path => ({ path, component })) }) From 84b983354152c85e9db9534dca29ac0b1f9ae91a Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:37:02 -0400 Subject: [PATCH 010/232] fix(web): send failed file downloads to the network page Downloads only logged a lost connection, so exporting a file while the server was unreachable left the page in place. They now go through the same failure handling as other requests. --- core/metaforge-web/src/utils/http.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/metaforge-web/src/utils/http.ts b/core/metaforge-web/src/utils/http.ts index 8c02673ebd..d17e69f630 100644 --- a/core/metaforge-web/src/utils/http.ts +++ b/core/metaforge-web/src/utils/http.ts @@ -154,6 +154,7 @@ export class HttpUtils return { status: true, blob, fileName: encoded ? decodeURIComponent(encoded) : plain || 'export' } }, error => { this.handlerMessage(error.message) + this.handlerFailed(error) return { status: false, message: String(error.message || '') } }) } From e053489261046fe64ce53f7b542a586da0e29152 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 22:59:18 -0400 Subject: [PATCH 011/232] feat(auth): let administrators turn on sign-in and sign-up features System settings (stored under the SYSTEM configure category) switch on registration, the captcha, remembered sign-ins, email sign-in, email verification on sign-up and password reset, and hold the SMTP server and the terms and privacy links. Email features only take effect once mail can be sent. The sign-in page reads what is on from /api/auth/options; codes are mailed by /api/auth/code and passwords reset by /api/auth/reset. Users get an email column, and administrators a System settings menu on fresh and existing installs. --- .../server/controller/CaptchaController.java | 9 +- .../controller/SystemSettingController.java | 66 +++++ .../controller/user/AuthController.java | 76 ++---- .../core/core-0007-system-settings.xml | 28 ++ .../db/changelog/db.changelog-master.xml | 1 + .../src/main/resources/db/seed/baseline.json | 17 +- .../resources/db/seed/system-settings.json | 23 ++ .../server/auth/AuthSettingsTest.java | 235 ++++++++++++++++ .../migration/DatabaseMigratorTest.java | 11 +- core/metaforge-service/pom.xml | 16 ++ .../metaforge/service/body/AuthBody.java | 8 +- .../metaforge/service/body/EmailCodeBody.java | 13 + .../service/body/PasswordResetBody.java | 14 + .../metaforge/service/entity/UserEntity.java | 4 + .../initializer/InitializerConfigure.java | 3 +- .../service/repository/UserRepository.java | 2 + .../service/service/AuthService.java | 24 ++ .../metaforge/service/service/JwtService.java | 9 +- .../service/service/SystemSettingService.java | 24 ++ .../service/service/UserService.java | 7 +- .../service/service/impl/AuthServiceImpl.java | 253 ++++++++++++++++++ .../impl/SystemSettingServiceImpl.java | 104 +++++++ .../service/service/impl/UserServiceImpl.java | 4 +- .../service/settings/AuthSettings.java | 60 +++++ .../service/settings/EmailCodes.java | 88 ++++++ .../service/settings/MailService.java | 10 + .../service/settings/MailSettings.java | 65 +++++ .../service/settings/SmtpMailService.java | 59 ++++ .../service/settings/SystemSettingSchema.java | 66 +++++ pom.xml | 2 + 30 files changed, 1237 insertions(+), 64 deletions(-) create mode 100644 core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java create mode 100644 core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml create mode 100644 core/metaforge-server/src/main/resources/db/seed/system-settings.json create mode 100644 core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SystemSettingServiceImpl.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java index 4f40d30048..4e5710c43e 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/CaptchaController.java @@ -1,24 +1,29 @@ package org.devlive.metaforge.server.controller; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.ObjectUtils; import org.devlive.metaforge.captcha.CalculateUtils; import org.devlive.metaforge.captcha.entity.ResultEntity; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.service.SystemSettingService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController() +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) @RequestMapping(value = "/api/v1/captcha") public class CaptchaController { private final InitializerConfigure initializer; + private final SystemSettingService settings; - public CaptchaController(InitializerConfigure initializer) + public CaptchaController(InitializerConfigure initializer, SystemSettingService settings) { this.initializer = initializer; + this.settings = settings; } @GetMapping @@ -28,7 +33,7 @@ public CommonResponse getCaptcha(@RequestParam(value = "timestamp") Long return CommonResponse.failure("timestamp is required"); } // If the verification code is disabled, directly skip the logic of obtaining the verification code - if (!initializer.getCaptchaEnable()) { + if (!settings.auth().isCaptcha()) { return CommonResponse.success(Boolean.FALSE); } ResultEntity entity = CalculateUtils.generate(); diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java new file mode 100644 index 0000000000..ba2e23b1f5 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SystemSettingController.java @@ -0,0 +1,66 @@ +package org.devlive.metaforge.server.controller; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.MailSettings; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** System settings, for administrators only (the /api/v1/admin/** guard). */ +@RestController +@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) +@RequestMapping(value = "/api/v1/admin/settings") +public class SystemSettingController +{ + private final SystemSettingService service; + + public SystemSettingController(SystemSettingService service) + { + this.service = service; + } + + @GetMapping + public CommonResponse> settings() + { + return CommonResponse.success(service.settings()); + } + + @PutMapping + public CommonResponse> save(@RequestBody SettingsBody body) + { + return service.save(body.getAuth(), body.getMail()); + } + + @PostMapping(value = "mail/test") + public CommonResponse testMail(@RequestBody MailTestBody body) + { + return service.testMail(body.getMail(), body.getTo()); + } + + @Data + @NoArgsConstructor + public static class SettingsBody + { + private AuthSettings auth; + private MailSettings mail; + } + + /** Tests the server as it is in the form, before it is saved. */ + @Data + @NoArgsConstructor + public static class MailTestBody + { + private MailSettings mail; + private String to; + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java index d8c77b129b..e071a6de3f 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/user/AuthController.java @@ -1,88 +1,62 @@ package org.devlive.metaforge.server.controller.user; import com.fasterxml.jackson.annotation.JsonView; -import org.apache.commons.lang3.ObjectUtils; -import org.devlive.metaforge.captcha.entity.ResultEntity; import org.devlive.metaforge.common.response.CommonResponse; import org.devlive.metaforge.common.view.EntityView; import org.devlive.metaforge.service.body.AuthBody; -import org.devlive.metaforge.service.entity.UserEntity; -import org.devlive.metaforge.service.initializer.InitializerConfigure; -import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.service.AuthService; import org.devlive.metaforge.service.validation.ValidationGroup; import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.Map; + @RestController @RequestMapping("/api/auth") public class AuthController { - private final UserService userService; - private final InitializerConfigure initializer; + private final AuthService authService; - public AuthController(UserService userService, InitializerConfigure initializer) + public AuthController(AuthService authService) { - this.userService = userService; - this.initializer = initializer; + this.authService = authService; } - /** - * Validates the captcha. - * - * @param configure the authentication body containing captcha information - * @return the common response object representing the result of the validation - */ - private CommonResponse validateCaptcha(AuthBody configure) + /** What the sign-in and sign-up pages offer, as set in system settings. */ + @GetMapping("/options") + public CommonResponse> options() { - if (initializer.getCaptchaEnable()) { - if (ObjectUtils.isEmpty(configure.getCaptcha())) { - return CommonResponse.failure("Captcha must not be empty"); - } - ResultEntity entity = initializer.getCache().getIfPresent(configure.getTimestamp()); - if (entity == null) { - return CommonResponse.failure("Captcha does not exists"); - } - if (!entity.getResult().equals(configure.getCaptcha())) { - initializer.getCache().invalidate(configure.getTimestamp()); - return CommonResponse.failure("The calculation result is incorrect"); - } - } - return null; + return CommonResponse.success(authService.options()); } @PostMapping("/signin") public CommonResponse authenticateUser(@RequestBody @Validated(ValidationGroup.Crud.Auth.class) AuthBody configure) { - CommonResponse captchaResponse = validateCaptcha(configure); - if (captchaResponse != null) { - return captchaResponse; - } - - UserEntity user = new UserEntity(); - user.setUsername(configure.getUsername()); - user.setPassword(configure.getPassword()); - return this.userService.authenticate(user); + return authService.signin(configure); } @PostMapping("/signup") @JsonView(value = {EntityView.UserView.class}) public CommonResponse registerUser(@RequestBody @Validated(ValidationGroup.Crud.Create.class) AuthBody configure) { - if (!initializer.getRegistrationEnable()) { - return CommonResponse.failure("Registration is disabled"); - } + return authService.signup(configure); + } - CommonResponse captchaResponse = validateCaptcha(configure); - if (captchaResponse != null) { - return captchaResponse; - } + @PostMapping("/code") + public CommonResponse sendCode(@RequestBody EmailCodeBody configure) + { + return authService.sendCode(configure); + } - UserEntity user = new UserEntity(); - user.setUsername(configure.getUsername()); - user.setPassword(configure.getPassword()); - return this.userService.saveOrUpdate(user); + @PostMapping("/reset") + public CommonResponse resetPassword(@RequestBody PasswordResetBody configure) + { + return authService.reset(configure); } } diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml new file mode 100644 index 0000000000..4501b2f1a8 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0007-system-settings.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS' + SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM' + SELECT COUNT(*) FROM metaforge_role WHERE code = 'ADMIN' + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml index 118fe3fa77..9db0dfaac0 100644 --- a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml +++ b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml @@ -32,4 +32,5 @@ + diff --git a/core/metaforge-server/src/main/resources/db/seed/baseline.json b/core/metaforge-server/src/main/resources/db/seed/baseline.json index 8118733a5b..0667f2ab9a 100644 --- a/core/metaforge-server/src/main/resources/db/seed/baseline.json +++ b/core/metaforge-server/src/main/resources/db/seed/baseline.json @@ -292,6 +292,20 @@ "icon": "Settings2", "isNew": false }, + { + "code": "SYSTEM_SETTINGS", + "name": "管理员 - 系统 - 系统设置", + "description": "管理员：管理员权限用户可以访问\\n位置：顶部管理一级子菜单", + "url": "/admin/settings", + "group": null, + "sorted": 7, + "type": "VIEW", + "parent": "SYSTEM", + "active": true, + "i18nKey": "navigation.systemSettings", + "icon": "SlidersHorizontal", + "isNew": false + }, { "code": "DATA_ANALYSIS", "name": "数据分析", @@ -326,7 +340,8 @@ "DATASET", "STORE", "b6669c359a7d41e581d906446366daf3", - "RUNTIME_CONFIGURE" + "RUNTIME_CONFIGURE", + "SYSTEM_SETTINGS" ], "USER": [ "HOME", diff --git a/core/metaforge-server/src/main/resources/db/seed/system-settings.json b/core/metaforge-server/src/main/resources/db/seed/system-settings.json new file mode 100644 index 0000000000..bab413d336 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/seed/system-settings.json @@ -0,0 +1,23 @@ +{ + "menus": [ + { + "code": "SYSTEM_SETTINGS", + "name": "管理员 - 系统 - 系统设置", + "description": "管理员：管理员权限用户可以访问\\n位置：顶部管理一级子菜单", + "url": "/admin/settings", + "group": null, + "sorted": 7, + "type": "VIEW", + "parent": "SYSTEM", + "active": true, + "i18nKey": "navigation.systemSettings", + "icon": "SlidersHorizontal", + "isNew": false + } + ], + "roleMenus": { + "ADMIN": [ + "SYSTEM_SETTINGS" + ] + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java new file mode 100644 index 0000000000..020cfc594a --- /dev/null +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/auth/AuthSettingsTest.java @@ -0,0 +1,235 @@ +package org.devlive.metaforge.server.auth; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.service.impl.AuthServiceImpl; +import org.devlive.metaforge.service.service.impl.SystemSettingServiceImpl; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.EmailCodes; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.security.crypto.password.PasswordEncoder; + +import javax.mail.MessagingException; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AuthSettingsTest +{ + private static final MailSettings MAIL = MailSettings.builder().host("smtp.example.com").port(465).ssl(true).from("noreply@example.com").password("secret").build(); + + private SystemSettingService settings; + private MailService mail; + private UserService users; + private UserRepository repository; + private EmailCodes codes; + private AuthServiceImpl service; + + @BeforeEach + public void setUp() + { + settings = mock(SystemSettingService.class); + mail = mock(MailService.class); + users = mock(UserService.class); + repository = mock(UserRepository.class); + codes = new EmailCodes(); + PasswordEncoder encoder = mock(PasswordEncoder.class); + when(encoder.encode(anyString())).thenAnswer(invocation -> "hash:" + invocation.getArgument(0)); + when(repository.findByEmail(anyString())).thenReturn(Optional.empty()); + service = new AuthServiceImpl(settings, codes, mail, users, repository, encoder, mock(InitializerConfigure.class)); + } + + private void given(AuthSettings auth, MailSettings server) + { + when(settings.auth()).thenReturn(auth); + when(settings.mail()).thenReturn(server); + } + + private static AuthSettings.AuthSettingsBuilder auth() + { + return AuthSettings.builder().registration(true).rememberDays(30); + } + + private static UserEntity user(String username, String email) + { + UserEntity user = new UserEntity(); + user.setUsername(username); + user.setEmail(email); + return user; + } + + @Test + public void emailFeaturesAreOfferedOnlyWhenTurnedOnAndMailCanBeSent() + { + given(auth().emailVerification(true).passwordReset(true).build(), new MailSettings()); + Map options = service.options(); + assertEquals(false, options.get("emailVerification")); + assertEquals(false, options.get("passwordReset")); + assertEquals(false, options.get("rememberMe")); + + given(auth().emailVerification(true).passwordReset(true).rememberMe(true).build(), MAIL); + options = service.options(); + assertEquals(true, options.get("emailVerification")); + assertEquals(true, options.get("passwordReset")); + assertEquals(true, options.get("rememberMe")); + assertFalse(service.sendCode(body("a@example.com", "unknown")).getStatus()); + } + + private static EmailCodeBody body(String email, String purpose) + { + EmailCodeBody body = new EmailCodeBody(); + body.setEmail(email); + body.setPurpose(purpose); + return body; + } + + @Test + public void signUpWithVerificationNeedsTheMailedCode() + throws MessagingException + { + given(auth().emailVerification(true).build(), MAIL); + when(users.saveOrUpdate(any(UserEntity.class))).thenReturn(CommonResponse.success(new UserEntity())); + assertTrue(service.sendCode(body("New@Example.com", "signup")).getStatus()); + ArgumentCaptor text = ArgumentCaptor.forClass(String.class); + verify(mail).send(eq(MAIL), eq("new@example.com"), anyString(), text.capture()); + String code = text.getValue().replaceAll("(?s).*?(\\d{6}).*", "$1"); + + // Asking again within a minute is refused. + assertFalse(service.sendCode(body("new@example.com", "signup")).getStatus()); + + AuthBody signup = new AuthBody(); + signup.setUsername("new_user"); + signup.setPassword("secret1"); + signup.setEmail("new@example.com"); + signup.setCode("000000".equals(code) ? "111111" : "000000"); + assertFalse(service.signup(signup).getStatus()); + signup.setCode(code); + assertTrue(service.signup(signup).getStatus()); + ArgumentCaptor saved = ArgumentCaptor.forClass(UserEntity.class); + verify(users).saveOrUpdate(saved.capture()); + assertEquals("new@example.com", saved.getValue().getEmail()); + // A code is used once. + assertFalse(service.signup(signup).getStatus()); + } + + @Test + public void signUpChecksTheSwitchAndTheUsernameRule() + { + given(auth().registration(false).build(), MAIL); + AuthBody signup = new AuthBody(); + signup.setUsername("valid_name"); + signup.setPassword("secret1"); + assertFalse(service.signup(signup).getStatus()); + + given(auth().build(), MAIL); + signup.setUsername("no spaces!"); + assertFalse(service.signup(signup).getStatus()); + verify(users, never()).saveOrUpdate(any(UserEntity.class)); + } + + @Test + public void codesAreNotMailedToUnknownAddressesOrKeptWhenSendingFails() + throws MessagingException + { + given(auth().passwordReset(true).build(), MAIL); + assertTrue(service.sendCode(body("nobody@example.com", "reset")).getStatus()); + verify(mail, never()).send(any(), anyString(), anyString(), anyString()); + + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(user("known", "known@example.com"))); + doThrow(new MessagingException("refused")).when(mail).send(any(), anyString(), anyString(), anyString()); + assertFalse(service.sendCode(body("known@example.com", "reset")).getStatus()); + assertEquals(0, codes.waiting("known@example.com", EmailCodes.Purpose.RESET)); + } + + @Test + public void aMailedCodeResetsThePassword() + { + given(auth().passwordReset(true).build(), MAIL); + UserEntity known = user("known", "known@example.com"); + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(known)); + String code = codes.issue("known@example.com", EmailCodes.Purpose.RESET); + + PasswordResetBody reset = new PasswordResetBody(); + reset.setEmail("known@example.com"); + reset.setCode(code); + reset.setPassword("short"); + assertFalse(service.reset(reset).getStatus()); + reset.setPassword("new-secret"); + assertTrue(service.reset(reset).getStatus()); + assertEquals("hash:new-secret", known.getPassword()); + verify(repository).save(known); + } + + @Test + public void signInByEmailAndRememberedSignInsLastTheSetDays() + { + given(auth().emailSignin(true).rememberMe(true).rememberDays(7).build(), new MailSettings()); + when(repository.findByEmail("known@example.com")).thenReturn(Optional.of(user("known", "known@example.com"))); + when(users.authenticate(any(UserEntity.class), anyLong())).thenReturn(CommonResponse.success("token")); + AuthBody signin = new AuthBody(); + signin.setUsername("Known@Example.com"); + signin.setPassword("secret1"); + signin.setRemember(true); + service.signin(signin); + ArgumentCaptor user = ArgumentCaptor.forClass(UserEntity.class); + verify(users).authenticate(user.capture(), eq(TimeUnit.DAYS.toMillis(7))); + assertEquals("known", user.getValue().getUsername()); + + // Without the setting, the box is ignored and the default lifetime applies. + given(auth().build(), new MailSettings()); + signin.setUsername("other"); + service.signin(signin); + verify(users).authenticate(any(UserEntity.class), eq(0L)); + } + + @Test + public void savedMailPasswordsAreKeptAndNeverSentBack() + { + RuntimeConfigureService configures = mock(RuntimeConfigureService.class); + InitializerConfigure initializer = mock(InitializerConfigure.class); + when(initializer.getRegistrationEnable()).thenReturn(true); + when(initializer.getCaptchaEnable()).thenReturn(false); + when(configures.getEffective(eq("SYSTEM"), eq("MAIL"), any())).thenReturn(MAIL.toMap()); + when(configures.getEffective(eq("SYSTEM"), eq("AUTH"), any())).thenReturn(Map.of("registration", "true")); + when(configures.save(anyString(), anyString(), anyMap(), any())).thenReturn(CommonResponse.success(null)); + SystemSettingServiceImpl system = new SystemSettingServiceImpl(configures, initializer, mail); + + MailSettings shown = (MailSettings) system.settings().get("mail"); + assertEquals("", shown.getPassword()); + assertTrue(shown.isPasswordSet()); + + assertTrue(system.save(auth().build(), MAIL.toBuilder().password("").host("smtp.other.com").build()).getStatus()); + @SuppressWarnings("unchecked") + ArgumentCaptor> values = ArgumentCaptor.forClass(Map.class); + verify(configures).save(eq("SYSTEM"), eq("MAIL"), values.capture(), any()); + assertEquals("secret", values.getValue().get("password")); + assertEquals("smtp.other.com", values.getValue().get("host")); + } +} diff --git a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java index 8fac9a8f48..a2a6563cdb 100644 --- a/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java +++ b/core/metaforge-server/src/test/java/org/devlive/metaforge/server/migration/DatabaseMigratorTest.java @@ -56,15 +56,16 @@ public void freshInstallCreatesSchemaAndSeedAndIsRepeatable() DatabaseMigrator.migrate(connection); JdbcTemplate jdbc = jdbc(connection); - assertEquals(20, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu")); + assertEquals(21, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu")); assertEquals(4, count(jdbc, "SELECT COUNT(*) FROM metaforge_role")); - assertEquals(28, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu")); + assertEquals(29, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu")); + assertEquals(count(jdbc, "SELECT id FROM metaforge_menu WHERE code = 'SYSTEM'"), count(jdbc, "SELECT parent FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS'")); assertEquals(2, count(jdbc, "SELECT COUNT(*) FROM metaforge_function")); assertEquals(2, count(jdbc, "SELECT COUNT(*) FROM metaforge_scheduled")); assertEquals("系统管理员", jdbc.queryForObject("SELECT name FROM metaforge_role WHERE code = 'ADMIN'", String.class)); assertEquals(count(jdbc, "SELECT id FROM metaforge_menu WHERE code = 'MANAGEMENT'"), count(jdbc, "SELECT parent FROM metaforge_menu WHERE code = 'QUERY'")); - assertEquals(18, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu rm JOIN metaforge_role r ON r.id = rm.role_id WHERE r.code = 'ADMIN'")); + assertEquals(19, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu rm JOIN metaforge_role r ON r.id = rm.role_id WHERE r.code = 'ADMIN'")); // Unicode text, booleans, timestamps and generated keys behave the same everywhere. jdbc.update("INSERT INTO metaforge_source (name, code, source_type, host, port, is_public, use_ssl, create_time) VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)", @@ -154,7 +155,9 @@ private void assertLegacyUpgrade(Connection legacy, Connection fresh, String sch assertEquals(1, count(jdbc, "SELECT COUNT(*) FROM metaforge_user_role WHERE user_id = 7 AND role_id = 1")); assertEquals(0, count(jdbc, "SELECT COUNT(*) FROM metaforge_user_role WHERE role_id = 999")); - assertEquals(18, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu WHERE role_id = 1")); + // the legacy menus plus System settings, added to existing installs by core-0007 + assertEquals(19, count(jdbc, "SELECT COUNT(*) FROM metaforge_role_menu WHERE role_id = 1")); + assertEquals(1, count(jdbc, "SELECT COUNT(*) FROM metaforge_menu WHERE code = 'SYSTEM_SETTINGS'")); assertEquals("Redis", jdbc.queryForObject("SELECT source_type FROM metaforge_source WHERE id = 11", String.class)); assertTrue(jdbc.queryForObject("SELECT is_public FROM metaforge_source WHERE id = 11", Boolean.class)); assertEquals(7, count(jdbc, "SELECT user_id FROM metaforge_dataset WHERE id = 21")); diff --git a/core/metaforge-service/pom.xml b/core/metaforge-service/pom.xml index 76a21215ee..c7f853dc12 100644 --- a/core/metaforge-service/pom.xml +++ b/core/metaforge-service/pom.xml @@ -83,6 +83,22 @@ + + com.sun.mail + jakarta.mail + ${jakarta-mail.version} + + + com.sun.activation + jakarta.activation + + + + + com.sun.activation + jakarta.activation + ${jakarta-activation.version} + io.jsonwebtoken jjwt diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java index 9081fdc49b..f3b281ddfa 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/AuthBody.java @@ -20,7 +20,8 @@ public class AuthBody ValidationGroup.Crud.Update.class, ValidationGroup.Crud.Auth.class }) - @Size(max = 20) + // An email when signing in by email; sign-up checks its own username rule. + @Size(max = 255) private String username; @NotBlank(groups = { @@ -32,4 +33,9 @@ public class AuthBody private String password; private Integer captcha; private Long timestamp; + private String email; + /** The email code on sign-up. */ + private String code; + /** Keep the sign-in for the remembered days when the system settings allow it. */ + private Boolean remember; } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java new file mode 100644 index 0000000000..7b554c5b37 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/EmailCodeBody.java @@ -0,0 +1,13 @@ +package org.devlive.metaforge.service.body; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** Asks for a code mailed to an address, for sign-up (SIGNUP) or a password reset (RESET). */ +@Data +@NoArgsConstructor +public class EmailCodeBody +{ + private String email; + private String purpose; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java new file mode 100644 index 0000000000..e250868bd5 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/body/PasswordResetBody.java @@ -0,0 +1,14 @@ +package org.devlive.metaforge.service.body; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** Sets a new password for the account of an email whose code was mailed. */ +@Data +@NoArgsConstructor +public class PasswordResetBody +{ + private String email; + private String code; + private String password; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java index 5dbc1c7341..e6b841ac4c 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/UserEntity.java @@ -66,6 +66,10 @@ public class UserEntity @JsonView(value = {EntityView.NoneView.class}) private String password; + @Column("email") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String email; + @Column("chat_configure") @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private String chatConfigure; diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java index 5c1f5d2ec4..a92a8c62dd 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/initializer/InitializerConfigure.java @@ -103,7 +103,8 @@ public void init() cacheExpiration = 2L; } log.info("Metaforge cache expiration: {}", this.cacheExpiration); - if (ObjectUtils.isEmpty(cache) && captchaEnable) { + // The captcha can be turned on in system settings at any time, so its cache always exists. + if (ObjectUtils.isEmpty(cache)) { cache = CacheBuilder.newBuilder() .expireAfterWrite(cacheExpiration, TimeUnit.MINUTES) .maximumSize(cacheMaximum) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java index c66fe850b8..4149d8dc46 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/UserRepository.java @@ -9,5 +9,7 @@ public interface UserRepository { Optional findByUsername(String username); + Optional findByEmail(String email); + UserEntity findByUsernameAndSystemTrue(String username); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java new file mode 100644 index 0000000000..2f37297c68 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/AuthService.java @@ -0,0 +1,24 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; + +import java.util.Map; + +/** Sign-in, sign-up and password reset, each feature only as the system settings allow. */ +public interface AuthService +{ + /** What the sign-in and sign-up pages offer; public. */ + Map options(); + + CommonResponse signin(AuthBody body); + + CommonResponse signup(AuthBody body); + + /** Mails a code; answers with the seconds to wait before asking again. */ + CommonResponse sendCode(EmailCodeBody body); + + CommonResponse reset(PasswordResetBody body); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java index 081dd64569..8f770868b7 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/JwtService.java @@ -42,12 +42,19 @@ public void warnOnDefaultSecret() } public String generateJwtToken(Authentication authentication) + { + return generateJwtToken(authentication, 0); + } + + /** A token lasting the given milliseconds, or metaforge.security.expiration when it is 0. */ + public String generateJwtToken(Authentication authentication, long expirationMs) { UserDetailsService userPrincipal = (UserDetailsService) authentication.getPrincipal(); + long lasting = expirationMs > 0 ? expirationMs : jwtExpirationMs; return Jwts.builder() .setSubject((userPrincipal.getUsername())) .setIssuedAt(new Date()) - .setExpiration(new Date((new Date()).getTime() + jwtExpirationMs)) + .setExpiration(new Date((new Date()).getTime() + lasting)) .signWith(SignatureAlgorithm.HS512, jwtSecret) .compact(); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java new file mode 100644 index 0000000000..bd58d71145 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/SystemSettingService.java @@ -0,0 +1,24 @@ +package org.devlive.metaforge.service.service; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.MailSettings; + +import java.util.Map; + +/** Settings an administrator changes in system settings: sign-in and registration features and the mail server. */ +public interface SystemSettingService +{ + AuthSettings auth(); + + MailSettings mail(); + + /** Both groups for the settings page, the mail password left out. */ + Map settings(); + + /** Saves both groups; an empty mail password keeps the stored one. */ + CommonResponse> save(AuthSettings auth, MailSettings mail); + + /** Sends a test mail with the saved server, or with the given one when it is passed. */ + CommonResponse testMail(MailSettings mail, String to); +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java index fcd7b35a7c..0a889c87a8 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/UserService.java @@ -20,7 +20,12 @@ public interface UserService { CommonResponse saveOrUpdate(UserEntity configure); - CommonResponse authenticate(UserEntity configure); + /** + * Signs in and returns the token. + * + * @param expiration how long the token lasts in milliseconds; 0 for the configured default + */ + CommonResponse authenticate(UserEntity configure, long expiration); CommonResponse info(String code); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java new file mode 100644 index 0000000000..677b178071 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/AuthServiceImpl.java @@ -0,0 +1,253 @@ +package org.devlive.metaforge.service.service.impl; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.captcha.entity.ResultEntity; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.AuthBody; +import org.devlive.metaforge.service.body.EmailCodeBody; +import org.devlive.metaforge.service.body.PasswordResetBody; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.service.AuthService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.service.UserService; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.EmailCodes; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; + +import javax.mail.MessagingException; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +@Slf4j +@Service +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class AuthServiceImpl + implements AuthService +{ + private static final Pattern EMAIL = Pattern.compile("^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$"); + private static final Pattern USERNAME = Pattern.compile("^[A-Za-z0-9_]{3,32}$"); + private static final int PASSWORD_MIN = 6; + private static final int PASSWORD_MAX = 120; + + private final SystemSettingService settings; + private final EmailCodes codes; + private final MailService mailService; + private final UserService userService; + private final UserRepository userRepository; + private final PasswordEncoder encoder; + private final InitializerConfigure initializer; + + public AuthServiceImpl(SystemSettingService settings, EmailCodes codes, MailService mailService, UserService userService, UserRepository userRepository, + PasswordEncoder encoder, InitializerConfigure initializer) + { + this.settings = settings; + this.codes = codes; + this.mailService = mailService; + this.userService = userService; + this.userRepository = userRepository; + this.encoder = encoder; + this.initializer = initializer; + } + + static boolean isEmail(String value) + { + return value != null && value.length() <= 255 && EMAIL.matcher(value.trim()).matches(); + } + + private static String normalize(String email) + { + return email.trim().toLowerCase(Locale.ROOT); + } + + /** Email verification and password reset only count once mail can be sent. */ + private static boolean verification(AuthSettings auth, MailSettings mail) + { + return auth.isRegistration() && auth.isEmailVerification() && mail.ready(); + } + + private static boolean reset(AuthSettings auth, MailSettings mail) + { + return auth.isPasswordReset() && mail.ready(); + } + + @Override + public Map options() + { + AuthSettings auth = settings.auth(); + MailSettings mail = settings.mail(); + Map options = new LinkedHashMap<>(); + options.put("registration", auth.isRegistration()); + options.put("captcha", auth.isCaptcha()); + options.put("rememberMe", auth.isRememberMe()); + options.put("rememberDays", auth.getRememberDays()); + options.put("emailSignin", auth.isEmailSignin()); + options.put("emailVerification", verification(auth, mail)); + options.put("passwordReset", reset(auth, mail)); + options.put("termsUrl", auth.getTermsUrl()); + options.put("privacyUrl", auth.getPrivacyUrl()); + // Third-party sign-in is listed here once a provider can be configured. + options.put("providers", Collections.emptyList()); + return options; + } + + private CommonResponse captcha(AuthSettings auth, AuthBody body) + { + if (!auth.isCaptcha()) { + return null; + } + if (ObjectUtils.isEmpty(body.getCaptcha()) || body.getTimestamp() == null) { + return CommonResponse.failure("Captcha must not be empty"); + } + ResultEntity entity = initializer.getCache().getIfPresent(body.getTimestamp()); + if (entity == null) { + return CommonResponse.failure("Captcha does not exists"); + } + if (!entity.getResult().equals(body.getCaptcha())) { + initializer.getCache().invalidate(body.getTimestamp()); + return CommonResponse.failure("The calculation result is incorrect"); + } + return null; + } + + @Override + public CommonResponse signin(AuthBody body) + { + AuthSettings auth = settings.auth(); + CommonResponse failed = captcha(auth, body); + if (failed != null) { + return failed; + } + String username = body.getUsername(); + if (auth.isEmailSignin() && isEmail(username)) { + Optional owner = userRepository.findByEmail(normalize(username)); + if (owner.isPresent()) { + username = owner.get().getUsername(); + } + } + long days = auth.isRememberMe() && Boolean.TRUE.equals(body.getRemember()) ? auth.getRememberDays() : 0; + UserEntity user = new UserEntity(); + user.setUsername(username); + user.setPassword(body.getPassword()); + return userService.authenticate(user, TimeUnit.DAYS.toMillis(days)); + } + + @Override + public CommonResponse signup(AuthBody body) + { + AuthSettings auth = settings.auth(); + if (!auth.isRegistration()) { + return CommonResponse.failure("Registration is disabled"); + } + CommonResponse failed = captcha(auth, body); + if (failed != null) { + return failed; + } + if (body.getUsername() == null || !USERNAME.matcher(body.getUsername()).matches()) { + return CommonResponse.failure("The username has 3 to 32 letters, digits or underscores"); + } + UserEntity user = new UserEntity(); + if (verification(auth, settings.mail())) { + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String email = normalize(body.getEmail()); + if (userRepository.findByEmail(email).isPresent()) { + return CommonResponse.failure("The email is already registered"); + } + if (!codes.verify(email, EmailCodes.Purpose.SIGNUP, body.getCode())) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + user.setEmail(email); + } + user.setUsername(body.getUsername()); + user.setPassword(body.getPassword()); + return userService.saveOrUpdate(user); + } + + @Override + public CommonResponse sendCode(EmailCodeBody body) + { + AuthSettings auth = settings.auth(); + MailSettings mail = settings.mail(); + EmailCodes.Purpose purpose; + try { + purpose = EmailCodes.Purpose.valueOf(StringUtils.defaultString(body.getPurpose()).toUpperCase(Locale.ROOT)); + } + catch (IllegalArgumentException e) { + return CommonResponse.failure("Unknown code purpose"); + } + boolean enabled = purpose == EmailCodes.Purpose.SIGNUP ? verification(auth, mail) : reset(auth, mail); + if (!enabled) { + return CommonResponse.failure("This feature is not enabled"); + } + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String email = normalize(body.getEmail()); + long waiting = codes.waiting(email, purpose); + if (waiting > 0) { + return CommonResponse.failure("Wait " + waiting + " seconds before asking for another code"); + } + boolean registered = userRepository.findByEmail(email).isPresent(); + if (purpose == EmailCodes.Purpose.SIGNUP && registered) { + return CommonResponse.failure("The email is already registered"); + } + String code = codes.issue(email, purpose); + // A reset for an unknown address answers the same way, so addresses can not be probed. + if (purpose == EmailCodes.Purpose.RESET && !registered) { + return CommonResponse.success(EmailCodes.RESEND_SECONDS); + } + try { + mailService.send(mail, email, "MetaForge 验证码 / Verification code", String.format( + "你的 MetaForge 验证码是 %s，%d 分钟内有效。如非本人操作请忽略本邮件。%nYour MetaForge verification code is %s. It expires in %d minutes; ignore this mail if you did not ask for it.", + code, EmailCodes.MINUTES, code, EmailCodes.MINUTES)); + } + catch (MessagingException | RuntimeException e) { + codes.discard(email, purpose); + log.warn("Sending a code to {} failed", email, e); + return CommonResponse.failure("Failed to send mail: " + e.getMessage()); + } + return CommonResponse.success(EmailCodes.RESEND_SECONDS); + } + + @Override + public CommonResponse reset(PasswordResetBody body) + { + if (!reset(settings.auth(), settings.mail())) { + return CommonResponse.failure("This feature is not enabled"); + } + if (!isEmail(body.getEmail())) { + return CommonResponse.failure("Enter a valid email address"); + } + String password = body.getPassword(); + if (password == null || password.length() < PASSWORD_MIN || password.length() > PASSWORD_MAX) { + return CommonResponse.failure("The password has 6 to 120 characters"); + } + String email = normalize(body.getEmail()); + if (!codes.verify(email, EmailCodes.Purpose.RESET, body.getCode())) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + Optional user = userRepository.findByEmail(email); + if (user.isEmpty()) { + return CommonResponse.failure("The email code is wrong or has expired"); + } + UserEntity value = user.get(); + value.setPassword(encoder.encode(password)); + userRepository.save(value); + return CommonResponse.success(Boolean.TRUE); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SystemSettingServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SystemSettingServiceImpl.java new file mode 100644 index 0000000000..3dbfc388d1 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/SystemSettingServiceImpl.java @@ -0,0 +1,104 @@ +package org.devlive.metaforge.service.service.impl; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.initializer.InitializerConfigure; +import org.devlive.metaforge.service.service.RuntimeConfigureService; +import org.devlive.metaforge.service.service.SystemSettingService; +import org.devlive.metaforge.service.settings.AuthSettings; +import org.devlive.metaforge.service.settings.MailService; +import org.devlive.metaforge.service.settings.MailSettings; +import org.devlive.metaforge.service.settings.SystemSettingSchema; +import org.springframework.stereotype.Service; + +import javax.mail.MessagingException; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Slf4j +@Service +public class SystemSettingServiceImpl + implements SystemSettingService +{ + private final RuntimeConfigureService configures; + private final InitializerConfigure initializer; + private final MailService mailService; + + public SystemSettingServiceImpl(RuntimeConfigureService configures, InitializerConfigure initializer, MailService mailService) + { + this.configures = configures; + this.initializer = initializer; + this.mailService = mailService; + } + + @Override + public AuthSettings auth() + { + return AuthSettings.from(configures.getEffective(SystemSettingSchema.CATEGORY, SystemSettingSchema.AUTH, + SystemSettingSchema.auth(Boolean.TRUE.equals(initializer.getRegistrationEnable()), Boolean.TRUE.equals(initializer.getCaptchaEnable())))); + } + + @Override + public MailSettings mail() + { + return MailSettings.from(configures.getEffective(SystemSettingSchema.CATEGORY, SystemSettingSchema.MAIL, SystemSettingSchema.mail())); + } + + @Override + public Map settings() + { + Map result = new LinkedHashMap<>(); + result.put("auth", auth()); + result.put("mail", mail().masked()); + return result; + } + + @Override + public CommonResponse> save(AuthSettings auth, MailSettings mail) + { + if (auth == null || mail == null) { + return CommonResponse.failure("Both setting groups are required"); + } + MailSettings saved = keepPassword(mail); + CommonResponse authSaved = configures.save(SystemSettingSchema.CATEGORY, SystemSettingSchema.AUTH, auth.toMap(), "Sign-in and registration"); + if (!Boolean.TRUE.equals(authSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(authSaved.getMessage())); + } + CommonResponse mailSaved = configures.save(SystemSettingSchema.CATEGORY, SystemSettingSchema.MAIL, saved.toMap(), "Mail server"); + if (!Boolean.TRUE.equals(mailSaved.getStatus())) { + return CommonResponse.failure(String.valueOf(mailSaved.getMessage())); + } + return CommonResponse.success(settings()); + } + + @Override + public CommonResponse testMail(MailSettings mail, String to) + { + MailSettings settings = mail == null ? mail() : keepPassword(mail); + if (!settings.ready()) { + return CommonResponse.failure("Set the mail server and the sender address first"); + } + if (!AuthServiceImpl.isEmail(to)) { + return CommonResponse.failure("Enter a valid email address"); + } + try { + mailService.send(settings, to.trim(), "MetaForge 测试邮件 / Test mail", + "这是一封来自 MetaForge 的测试邮件，邮件服务配置可用。\nThis is a test mail from MetaForge; the mail server works."); + return CommonResponse.success(Boolean.TRUE); + } + catch (MessagingException | RuntimeException e) { + log.warn("Test mail to {} failed", to, e); + return CommonResponse.failure("Failed to send mail: " + e.getMessage()); + } + } + + private MailSettings keepPassword(MailSettings mail) + { + if (StringUtils.isNotEmpty(mail.getPassword())) { + return mail; + } + return mail.toBuilder().password(mail().getPassword()).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java index 40ae5b8ec7..6db61ddce4 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/UserServiceImpl.java @@ -208,13 +208,13 @@ public CommonResponse saveOrUpdate(BaseRepository @AuditUserLog @Override - public CommonResponse authenticate(UserEntity configure) + public CommonResponse authenticate(UserEntity configure, long expiration) { UsernamePasswordAuthenticationToken authenticationToken = new UsernamePasswordAuthenticationToken(configure.getUsername(), configure.getPassword()); Authentication authentication = authenticationManager.authenticate(authenticationToken); SecurityContextHolder.getContext().setAuthentication(authentication); - String jwt = jwtService.generateJwtToken(authentication); + String jwt = jwtService.generateJwtToken(authentication, expiration); UserDetailsService userDetails = (UserDetailsService) authentication.getPrincipal(); List roles = userDetails.getAuthorities().stream() diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java new file mode 100644 index 0000000000..26ea5581da --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/AuthSettings.java @@ -0,0 +1,60 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Sign-in and registration features an administrator turns on in the system settings. + * The email features also need a configured mail server; see {@link SystemSettingSchema}. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AuthSettings +{ + private boolean registration; + private boolean captcha; + private boolean rememberMe; + private int rememberDays; + private boolean emailSignin; + private boolean emailVerification; + private boolean passwordReset; + private String termsUrl; + private String privacyUrl; + + public static AuthSettings from(Map values) + { + return AuthSettings.builder() + .registration(Boolean.parseBoolean(values.get("registration"))) + .captcha(Boolean.parseBoolean(values.get("captcha"))) + .rememberMe(Boolean.parseBoolean(values.get("rememberMe"))) + .rememberDays(SystemSettingSchema.number(values.get("rememberDays"), 30, 1, 365)) + .emailSignin(Boolean.parseBoolean(values.get("emailSignin"))) + .emailVerification(Boolean.parseBoolean(values.get("emailVerification"))) + .passwordReset(Boolean.parseBoolean(values.get("passwordReset"))) + .termsUrl(SystemSettingSchema.text(values.get("termsUrl"))) + .privacyUrl(SystemSettingSchema.text(values.get("privacyUrl"))) + .build(); + } + + public Map toMap() + { + Map values = new LinkedHashMap<>(); + values.put("registration", String.valueOf(registration)); + values.put("captcha", String.valueOf(captcha)); + values.put("rememberMe", String.valueOf(rememberMe)); + values.put("rememberDays", String.valueOf(Math.max(1, Math.min(365, rememberDays)))); + values.put("emailSignin", String.valueOf(emailSignin)); + values.put("emailVerification", String.valueOf(emailVerification)); + values.put("passwordReset", String.valueOf(passwordReset)); + values.put("termsUrl", SystemSettingSchema.text(termsUrl)); + values.put("privacyUrl", SystemSettingSchema.text(privacyUrl)); + return values; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java new file mode 100644 index 0000000000..656cf303ec --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/EmailCodes.java @@ -0,0 +1,88 @@ +package org.devlive.metaforge.service.settings; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; + +import java.security.SecureRandom; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Six-digit codes mailed for sign-up and password reset. A code lasts ten minutes, a new one is + * sent at most once a minute per address, and five wrong tries discard it. + */ +@Component +public class EmailCodes +{ + public static final int MINUTES = 10; + public static final long RESEND_SECONDS = 60; + private static final int ATTEMPTS = 5; + + private final SecureRandom random = new SecureRandom(); + private final Cache codes = CacheBuilder.newBuilder() + .expireAfterWrite(MINUTES, TimeUnit.MINUTES) + .maximumSize(10000) + .build(); + + public enum Purpose + { + SIGNUP, RESET + } + + @AllArgsConstructor + private static final class Entry + { + private final String code; + private final long sent; + private int attempts; + } + + /** Seconds before another code may be sent to the address; 0 when it may be sent now. */ + public long waiting(String email, Purpose purpose) + { + Entry entry = codes.getIfPresent(key(email, purpose)); + if (entry == null) { + return 0; + } + long passed = (System.currentTimeMillis() - entry.sent) / 1000; + return Math.max(0, RESEND_SECONDS - passed); + } + + /** A new code for the address, replacing any earlier one. */ + public String issue(String email, Purpose purpose) + { + String code = String.format("%06d", random.nextInt(1_000_000)); + codes.put(key(email, purpose), new Entry(code, System.currentTimeMillis(), 0)); + return code; + } + + public void discard(String email, Purpose purpose) + { + codes.invalidate(key(email, purpose)); + } + + /** Checks a code; a matching code is used up, and too many wrong ones discard it. */ + public synchronized boolean verify(String email, Purpose purpose, String code) + { + String key = key(email, purpose); + Entry entry = codes.getIfPresent(key); + if (entry == null || code == null) { + return false; + } + if (entry.code.equals(code.trim())) { + codes.invalidate(key); + return true; + } + if (++entry.attempts >= ATTEMPTS) { + codes.invalidate(key); + } + return false; + } + + private static String key(String email, Purpose purpose) + { + return purpose.name() + ":" + email.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java new file mode 100644 index 0000000000..5017ea8e89 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailService.java @@ -0,0 +1,10 @@ +package org.devlive.metaforge.service.settings; + +import javax.mail.MessagingException; + +/** Sends a plain text mail through the configured SMTP server. */ +public interface MailService +{ + void send(MailSettings settings, String to, String subject, String text) + throws MessagingException; +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java new file mode 100644 index 0000000000..3a5f1144c2 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/MailSettings.java @@ -0,0 +1,65 @@ +package org.devlive.metaforge.service.settings; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** The SMTP server that sends verification codes. */ +@Data +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class MailSettings +{ + private String host; + private int port; + private boolean ssl; + private String username; + private String password; + private String from; + /** Read only: whether a password is stored, since the password itself is never sent back. */ + private boolean passwordSet; + + public static MailSettings from(Map values) + { + String password = SystemSettingSchema.text(values.get("password")); + return MailSettings.builder() + .host(SystemSettingSchema.text(values.get("host"))) + .port(SystemSettingSchema.number(values.get("port"), 465, 1, 65535)) + .ssl(Boolean.parseBoolean(values.get("ssl"))) + .username(SystemSettingSchema.text(values.get("username"))) + .password(password) + .from(SystemSettingSchema.text(values.get("from"))) + .passwordSet(!password.isEmpty()) + .build(); + } + + public Map toMap() + { + Map values = new LinkedHashMap<>(); + values.put("host", SystemSettingSchema.text(host)); + values.put("port", String.valueOf(Math.max(1, Math.min(65535, port)))); + values.put("ssl", String.valueOf(ssl)); + values.put("username", SystemSettingSchema.text(username)); + values.put("password", SystemSettingSchema.text(password)); + values.put("from", SystemSettingSchema.text(from)); + return values; + } + + /** Mail can be sent once a server and a sender address are set. */ + public boolean ready() + { + return StringUtils.isNotBlank(host) && StringUtils.isNotBlank(from); + } + + /** The settings as the administrator sees them: everything except the password. */ + public MailSettings masked() + { + return toBuilder().password("").passwordSet(StringUtils.isNotEmpty(password)).build(); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java new file mode 100644 index 0000000000..88eae12ac0 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SmtpMailService.java @@ -0,0 +1,59 @@ +package org.devlive.metaforge.service.settings; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +@Component +public class SmtpMailService + implements MailService +{ + private static final String TIMEOUT = "10000"; + + @Override + public void send(MailSettings settings, String to, String subject, String text) + throws MessagingException + { + Properties properties = new Properties(); + properties.put("mail.transport.protocol", "smtp"); + properties.put("mail.smtp.host", settings.getHost()); + properties.put("mail.smtp.port", String.valueOf(settings.getPort())); + properties.put("mail.smtp.connectiontimeout", TIMEOUT); + properties.put("mail.smtp.timeout", TIMEOUT); + properties.put("mail.smtp.writetimeout", TIMEOUT); + if (settings.isSsl()) { + properties.put("mail.smtp.ssl.enable", "true"); + } + else { + properties.put("mail.smtp.starttls.enable", "true"); + } + boolean auth = StringUtils.isNotEmpty(settings.getUsername()); + properties.put("mail.smtp.auth", String.valueOf(auth)); + Session session = Session.getInstance(properties, auth ? new Authenticator() + { + @Override + protected PasswordAuthentication getPasswordAuthentication() + { + return new PasswordAuthentication(settings.getUsername(), settings.getPassword()); + } + } : null); + + MimeMessage message = new MimeMessage(session); + message.setFrom(new InternetAddress(settings.getFrom())); + message.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to)); + message.setSubject(subject, StandardCharsets.UTF_8.name()); + message.setText(text, StandardCharsets.UTF_8.name()); + Transport.send(message); + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java new file mode 100644 index 0000000000..304bb85a67 --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/settings/SystemSettingSchema.java @@ -0,0 +1,66 @@ +package org.devlive.metaforge.service.settings; + +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureField; +import org.devlive.metaforge.plugin.configure.RuntimeConfigureFieldType; + +import java.util.Arrays; +import java.util.List; + +/** + * The system settings stored in metaforge_configure under the SYSTEM category. + * Registration and captcha default to metaforge.registration.enable and metaforge.captcha.enable; + * every other feature is off until an administrator turns it on. + */ +public final class SystemSettingSchema +{ + public static final String CATEGORY = "SYSTEM"; + public static final String AUTH = "AUTH"; + public static final String MAIL = "MAIL"; + + private SystemSettingSchema() + {} + + public static List auth(boolean registration, boolean captcha) + { + return Arrays.asList( + new RuntimeConfigureField("registration", RuntimeConfigureFieldType.BOOLEAN, String.valueOf(registration), "Allow new accounts to sign up"), + new RuntimeConfigureField("captcha", RuntimeConfigureFieldType.BOOLEAN, String.valueOf(captcha), "Ask for the arithmetic captcha on sign-in and sign-up"), + new RuntimeConfigureField("rememberMe", RuntimeConfigureFieldType.BOOLEAN, "false", "Offer to keep the sign-in for several days"), + new RuntimeConfigureField("rememberDays", RuntimeConfigureFieldType.NUMBER, "30", "Days a remembered sign-in lasts"), + new RuntimeConfigureField("emailSignin", RuntimeConfigureFieldType.BOOLEAN, "false", "Allow signing in with the account email"), + new RuntimeConfigureField("emailVerification", RuntimeConfigureFieldType.BOOLEAN, "false", "Require an email and its code to sign up"), + new RuntimeConfigureField("passwordReset", RuntimeConfigureFieldType.BOOLEAN, "false", "Allow resetting a forgotten password by email"), + new RuntimeConfigureField("termsUrl", RuntimeConfigureFieldType.STRING, "", "Terms of service shown on sign-up"), + new RuntimeConfigureField("privacyUrl", RuntimeConfigureFieldType.STRING, "", "Privacy policy shown on sign-up")); + } + + public static List mail() + { + return Arrays.asList( + new RuntimeConfigureField("host", RuntimeConfigureFieldType.STRING, "", "SMTP host"), + new RuntimeConfigureField("port", RuntimeConfigureFieldType.NUMBER, "465", "SMTP port"), + new RuntimeConfigureField("ssl", RuntimeConfigureFieldType.BOOLEAN, "true", "SSL on connect; STARTTLS otherwise"), + new RuntimeConfigureField("username", RuntimeConfigureFieldType.STRING, "", "SMTP account"), + new RuntimeConfigureField("password", RuntimeConfigureFieldType.PASSWORD, "", "SMTP password"), + new RuntimeConfigureField("from", RuntimeConfigureFieldType.STRING, "", "Sender address")); + } + + static String text(String value) + { + return value == null ? "" : value.trim(); + } + + static int number(String value, int fallback, int min, int max) + { + if (StringUtils.isBlank(value)) { + return fallback; + } + try { + return Math.max(min, Math.min(max, (int) Double.parseDouble(value.trim()))); + } + catch (NumberFormatException e) { + return fallback; + } + } +} diff --git a/pom.xml b/pom.xml index c18379699e..31ad92995a 100644 --- a/pom.xml +++ b/pom.xml @@ -203,6 +203,8 @@ 3.8.0 1.33 1.21 + 1.6.7 + 1.2.2 2024.01.3 1.20.1 2.3.2 From 4ea74b0d94ed188debbd4a88a2c146c0f3d1ccfd Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 23:09:01 -0400 Subject: [PATCH 012/232] feat(web): add the System settings page Administrators turn sign-in and sign-up features on and off, set the agreement links and the SMTP server, and send a test mail with the server as it is in the form. Features that send mail are marked until a mail server is set. --- core/metaforge-web/src/i18n/locales/en.json | 46 +++++ .../metaforge-web/src/i18n/locales/zh-cn.json | 46 +++++ .../metaforge-web/src/model/system-setting.ts | 54 ++++++ core/metaforge-web/src/router/default.ts | 9 + core/metaforge-web/src/router/navigation.ts | 2 +- .../src/services/system-setting.ts | 26 +++ .../pages/system/settings/SettingsHome.vue | 160 ++++++++++++++++++ .../tests/system-setting.test.mjs | 33 ++++ 8 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 core/metaforge-web/src/model/system-setting.ts create mode 100644 core/metaforge-web/src/services/system-setting.ts create mode 100644 core/metaforge-web/src/views/pages/system/settings/SettingsHome.vue create mode 100644 core/metaforge-web/tests/system-setting.test.mjs diff --git a/core/metaforge-web/src/i18n/locales/en.json b/core/metaforge-web/src/i18n/locales/en.json index 44312d53db..c4ff4acf69 100644 --- a/core/metaforge-web/src/i18n/locales/en.json +++ b/core/metaforge-web/src/i18n/locales/en.json @@ -1274,6 +1274,7 @@ "navigation.pageNotAuthorized": "Not authorized", "navigation.pageNotFound": "Page not found", "navigation.systemManagement": "Administration", + "navigation.systemSettings": "System settings", "notify.common.apply": "Apply", "notify.common.batch": "Batch Settings", "notify.common.channels": "Notification Channels", @@ -1419,6 +1420,51 @@ "schedule.type.SOURCE_CHECK": "Source check", "schedule.type.SOURCE_SYNCHRONIZE": "Source synchronize", "schedule.type.SOURCE_SYNC_METADATA": "Metadata sync", + "settings.auth.agreementHint": "Once either address is set, sign-up asks to agree to it.", + "settings.auth.captcha": "Captcha", + "settings.auth.captchaHint": "Ask for an arithmetic answer on sign-in and sign-up", + "settings.auth.days": "days", + "settings.auth.description": "Features that are off do not appear on the sign-in and sign-up pages.", + "settings.auth.emailSignin": "Email sign-in", + "settings.auth.emailSigninHint": "Sign in with the account email", + "settings.auth.emailVerification": "Email verification", + "settings.auth.emailVerificationHint": "Sign-up needs an email and the code mailed to it", + "settings.auth.mailMissing": "Needs mail", + "settings.auth.mailReady": "Mail ready", + "settings.auth.needsMail": "{features} take effect once the mail server is set.", + "settings.auth.passwordReset": "Password reset", + "settings.auth.passwordResetHint": "Reset a forgotten password with an emailed code", + "settings.auth.privacyUrl": "Privacy policy URL", + "settings.auth.registration": "Open sign-up", + "settings.auth.registrationHint": "Visitors can create their own accounts", + "settings.auth.rememberMe": "Remember sign-in", + "settings.auth.rememberMeHint": "Keep users signed in for the set days when they ask", + "settings.auth.termsUrl": "Terms of service URL", + "settings.auth.title": "Sign-in and sign-up", + "settings.auth.urlInvalid": "Enter an address starting with http:// or https://", + "settings.description": "Manage sign-in, sign-up and the mail server; a feature appears on the sign-in page only once it is on.", + "settings.mail.description": "Sends sign-up and password reset codes (SMTP).", + "settings.mail.from": "Sender address", + "settings.mail.fromInvalid": "Enter a valid sender email", + "settings.mail.host": "SMTP host", + "settings.mail.hostRequired": "Mail features are on; enter the SMTP host", + "settings.mail.notReady": "Not set", + "settings.mail.password": "Password / app password", + "settings.mail.passwordKept": "Saved; leave empty to keep it", + "settings.mail.port": "Port", + "settings.mail.portRequired": "Enter the port", + "settings.mail.ready": "Set", + "settings.mail.ssl": "SSL", + "settings.mail.sslHint": "STARTTLS when off (usually port 587)", + "settings.mail.test": "Send a test mail", + "settings.mail.testHint": "Uses the settings in the form; no need to save first.", + "settings.mail.testSend": "Send", + "settings.mail.testSent": "Test mail sent to {to}", + "settings.mail.testTo": "Recipient email", + "settings.mail.title": "Mail server", + "settings.mail.username": "Account", + "settings.save": "Save", + "settings.saved": "Settings saved", "snippet.common.create": "Create Snippet", "snippet.common.delete": "Delete Snippet", "snippet.common.deleteInfo": "Delete Snippet [ $VALUE ]", diff --git a/core/metaforge-web/src/i18n/locales/zh-cn.json b/core/metaforge-web/src/i18n/locales/zh-cn.json index 8e247883cc..cd30ad9847 100644 --- a/core/metaforge-web/src/i18n/locales/zh-cn.json +++ b/core/metaforge-web/src/i18n/locales/zh-cn.json @@ -1274,6 +1274,7 @@ "navigation.pageNotAuthorized": "无权访问", "navigation.pageNotFound": "页面未找到", "navigation.systemManagement": "系统管理", + "navigation.systemSettings": "系统设置", "notify.common.apply": "应用", "notify.common.batch": "批量设置", "notify.common.channels": "通知渠道", @@ -1419,6 +1420,51 @@ "schedule.type.SOURCE_CHECK": "数据源检查", "schedule.type.SOURCE_SYNCHRONIZE": "数据源同步", "schedule.type.SOURCE_SYNC_METADATA": "元数据同步", + "settings.auth.agreementHint": "填写任一地址后，注册页会要求勾选同意协议。", + "settings.auth.captcha": "图形验证码", + "settings.auth.captchaHint": "登录、注册时需要输入计算结果", + "settings.auth.days": "天", + "settings.auth.description": "关闭的功能不会出现在登录、注册页面。", + "settings.auth.emailSignin": "邮箱登录", + "settings.auth.emailSigninHint": "允许使用账号绑定的邮箱登录", + "settings.auth.emailVerification": "注册邮箱验证", + "settings.auth.emailVerificationHint": "注册时需填写邮箱并输入邮件验证码", + "settings.auth.mailMissing": "需要邮件服务", + "settings.auth.mailReady": "邮件服务可用", + "settings.auth.needsMail": "{features}需要配置邮件服务后才会生效。", + "settings.auth.passwordReset": "找回密码", + "settings.auth.passwordResetHint": "通过邮箱验证码重置密码", + "settings.auth.privacyUrl": "隐私政策地址", + "settings.auth.registration": "开放注册", + "settings.auth.registrationHint": "允许访客自行注册账号", + "settings.auth.rememberMe": "记住登录状态", + "settings.auth.rememberMeHint": "勾选后在设定天数内保持登录", + "settings.auth.termsUrl": "服务协议地址", + "settings.auth.title": "登录与注册", + "settings.auth.urlInvalid": "请输入以 http:// 或 https:// 开头的地址", + "settings.description": "管理登录注册方式与邮件服务；功能只在开启后出现在登录页。", + "settings.mail.description": "用于发送注册验证码和找回密码邮件（SMTP）。", + "settings.mail.from": "发件地址", + "settings.mail.fromInvalid": "请输入正确的发件邮箱", + "settings.mail.host": "SMTP 服务器", + "settings.mail.hostRequired": "已开启邮件相关功能，请填写 SMTP 服务器", + "settings.mail.notReady": "未配置", + "settings.mail.password": "密码 / 授权码", + "settings.mail.passwordKept": "已保存，留空则不修改", + "settings.mail.port": "端口", + "settings.mail.portRequired": "请输入端口", + "settings.mail.ready": "已配置", + "settings.mail.ssl": "SSL 加密", + "settings.mail.sslHint": "关闭时使用 STARTTLS（常见端口 587）", + "settings.mail.test": "发送测试邮件", + "settings.mail.testHint": "使用当前填写的配置发送，无需先保存。", + "settings.mail.testSend": "发送", + "settings.mail.testSent": "测试邮件已发送至 {to}", + "settings.mail.testTo": "收件邮箱", + "settings.mail.title": "邮件服务", + "settings.mail.username": "账号", + "settings.save": "保存", + "settings.saved": "设置已保存", "snippet.common.create": "创建片段", "snippet.common.delete": "删除片段", "snippet.common.deleteInfo": "删除片段 [ $VALUE ]", diff --git a/core/metaforge-web/src/model/system-setting.ts b/core/metaforge-web/src/model/system-setting.ts new file mode 100644 index 0000000000..fbdbf6d71c --- /dev/null +++ b/core/metaforge-web/src/model/system-setting.ts @@ -0,0 +1,54 @@ +/** Sign-in and registration features an administrator turns on in system settings. */ +export interface AuthSettings { + registration: boolean + captcha: boolean + rememberMe: boolean + rememberDays: number + emailSignin: boolean + emailVerification: boolean + passwordReset: boolean + termsUrl: string + privacyUrl: string +} + +/** The SMTP server; the password is never sent back, `passwordSet` tells whether one is stored. */ +export interface MailSettings { + host: string + port: number + ssl: boolean + username: string + password: string + from: string + passwordSet?: boolean +} + +/** What the sign-in pages offer: the switches as they take effect (email ones need a mail server). */ +export interface AuthOptions extends AuthSettings { + providers: string[] +} + +export type CodePurpose = 'SIGNUP' | 'RESET' + +export const EMAIL_PATTERN = /^[^@\s]+@[^@\s]+\.[^@\s]+$/ +export const USERNAME_PATTERN = /^[A-Za-z0-9_]{3,32}$/ +export const URL_PATTERN = /^https?:\/\/\S+$/i + +export const defaultAuthSettings = (): AuthSettings => ({ + registration: true, captcha: true, rememberMe: false, rememberDays: 30, emailSignin: false, emailVerification: false, passwordReset: false, termsUrl: '', privacyUrl: '', +}) + +export const defaultMailSettings = (): MailSettings => ({ host: '', port: 465, ssl: true, username: '', password: '', from: '', passwordSet: false }) + +/** Email verification and password reset only work once mail can be sent. */ +export const mailReady = (mail: MailSettings) => !!mail.host.trim() && !!mail.from.trim() + +/** The switches that need a mail server but have none yet. */ +export function waitingForMail(auth: AuthSettings, mail: MailSettings): ('emailVerification' | 'passwordReset')[] { + if (mailReady(mail)) return [] + return (['emailVerification', 'passwordReset'] as const).filter(key => auth[key]) +} + +/** The agreement links shown on sign-up, in order; the box is required once there is one. */ +export function agreements(options: Pick): { key: 'terms' | 'privacy'; url: string }[] { + return [{ key: 'terms' as const, url: options.termsUrl }, { key: 'privacy' as const, url: options.privacyUrl }].filter(item => URL_PATTERN.test(item.url || '')) +} diff --git a/core/metaforge-web/src/router/default.ts b/core/metaforge-web/src/router/default.ts index b6f538d1a3..0a84d9c1e7 100644 --- a/core/metaforge-web/src/router/default.ts +++ b/core/metaforge-web/src/router/default.ts @@ -268,6 +268,15 @@ const createDefaultRouter = (router: Router) => { isRoot: false }, component: () => import('@/views/pages/system/configure/ConfigureHome.vue') + }, + { + name: 'settings', + path: 'settings', + meta: { + title: 'navigation.systemSettings', + isRoot: false + }, + component: () => import('@/views/pages/system/settings/SettingsHome.vue') } ] }) diff --git a/core/metaforge-web/src/router/navigation.ts b/core/metaforge-web/src/router/navigation.ts index c664c2a76c..6eafd5f34b 100644 --- a/core/metaforge-web/src/router/navigation.ts +++ b/core/metaforge-web/src/router/navigation.ts @@ -6,7 +6,7 @@ export function canonicalPagePath(path: string): string { result = result.replace(/^\/admin(?=[/?#]|$)/i, '/admin') result = result.replace(/^\/admin\/user\/(profile|info|log|editor|assistant|username|password|notify)(?=[/?#]|$)/i, (_match, page) => `/account/${page.toLowerCase()}`) result = result.replace(/^\/admin\/(query|source|dataset|dashboard|history|snippet|report|chat|workflow)(?=[/?#]|$)/i, (_match, page) => `/${page.toLowerCase()}`) - return result.replace(/^\/admin\/(user|role|menu|schedule|function|configure)(?=[/?#]|$)/i, (_match, page) => `/admin/${page.toLowerCase()}`) + return result.replace(/^\/admin\/(user|role|menu|schedule|function|configure|settings)(?=[/?#]|$)/i, (_match, page) => `/admin/${page.toLowerCase()}`) } export function isAdministrator(user: { roles?: unknown } | null | undefined): boolean { diff --git a/core/metaforge-web/src/services/system-setting.ts b/core/metaforge-web/src/services/system-setting.ts new file mode 100644 index 0000000000..9a7d481d84 --- /dev/null +++ b/core/metaforge-web/src/services/system-setting.ts @@ -0,0 +1,26 @@ +import { ResponseModel } from '@/model/response' +import { HttpUtils } from '@/utils/http' +import type { AuthSettings, MailSettings } from '@/model/system-setting' + +const DEFAULT_PATH = '/api/v1/admin/settings' + +class SystemSettingService +{ + get(): Promise + { + return new HttpUtils().get(DEFAULT_PATH) + } + + save(auth: AuthSettings, mail: MailSettings): Promise + { + return new HttpUtils().put(DEFAULT_PATH, { auth, mail }) + } + + /** Sends a test mail with the server as it is in the form. */ + testMail(mail: MailSettings, to: string): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/mail/test`, { mail, to }) + } +} + +export default new SystemSettingService() diff --git a/core/metaforge-web/src/views/pages/system/settings/SettingsHome.vue b/core/metaforge-web/src/views/pages/system/settings/SettingsHome.vue new file mode 100644 index 0000000000..0e21df3ff3 --- /dev/null +++ b/core/metaforge-web/src/views/pages/system/settings/SettingsHome.vue @@ -0,0 +1,160 @@ + + + diff --git a/core/metaforge-web/tests/system-setting.test.mjs b/core/metaforge-web/tests/system-setting.test.mjs new file mode 100644 index 0000000000..2834fe45c1 --- /dev/null +++ b/core/metaforge-web/tests/system-setting.test.mjs @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' +import ts from 'typescript' +const source = await readFile(new URL('../src/model/system-setting.ts', import.meta.url), 'utf8') +const { outputText } = ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2020 } }) +const { agreements, defaultAuthSettings, defaultMailSettings, EMAIL_PATTERN, mailReady, USERNAME_PATTERN, waitingForMail } = await import(`data:text/javascript;base64,${ Buffer.from(outputText).toString('base64') }`) + +test('mail counts as ready once a host and a sender are set', () => { + assert.equal(mailReady(defaultMailSettings()), false) + assert.equal(mailReady({ ...defaultMailSettings(), host: 'smtp.example.com' }), false) + assert.equal(mailReady({ ...defaultMailSettings(), host: ' smtp.example.com ', from: 'noreply@example.com' }), true) +}) + +test('only the mail features that are on wait for a mail server', () => { + const auth = { ...defaultAuthSettings(), emailVerification: true, emailSignin: true } + assert.deepEqual(waitingForMail(auth, defaultMailSettings()), ['emailVerification']) + assert.deepEqual(waitingForMail({ ...auth, passwordReset: true }, defaultMailSettings()), ['emailVerification', 'passwordReset']) + assert.deepEqual(waitingForMail(auth, { ...defaultMailSettings(), host: 'smtp.example.com', from: 'a@example.com' }), []) +}) + +test('sign-up asks to agree only to the agreements with an address', () => { + assert.deepEqual(agreements({ termsUrl: '', privacyUrl: '' }), []) + assert.deepEqual(agreements({ termsUrl: 'https://example.com/terms', privacyUrl: 'not a url' }), [{ key: 'terms', url: 'https://example.com/terms' }]) + assert.deepEqual(agreements({ termsUrl: 'https://a.example/t', privacyUrl: 'http://a.example/p' }).map(item => item.key), ['terms', 'privacy']) +}) + +test('usernames and emails follow the sign-up rules', () => { + for (const name of ['abc', 'user_01', 'A'.repeat(32)]) assert.equal(USERNAME_PATTERN.test(name), true, name) + for (const name of ['ab', 'A'.repeat(33), 'with space', 'dash-name', '中文名']) assert.equal(USERNAME_PATTERN.test(name), false, name) + assert.equal(EMAIL_PATTERN.test('a@example.com'), true) + assert.equal(EMAIL_PATTERN.test('a@example'), false) +}) From 85c18b36eb57e105a7ec34b244368dd7b4f5bd42 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 23:09:08 -0400 Subject: [PATCH 013/232] feat(web): rebuild sign-in and sign-up after the prototype The auth pages share a brand panel and a card with a language menu and sign-in / sign-up tabs. Each optional part appears only when system settings turn it on: the captcha, remembered sign-in, sign-in by email, the forgot password link and its reset page, email verification with a code button that counts down, and the agreement box. Sign-up shows that it is closed when registration is off. --- .../src/composables/useAuthCaptcha.ts | 19 ++ .../src/composables/useAuthOptions.ts | 21 ++ .../src/composables/useEmailCode.ts | 39 +++ core/metaforge-web/src/i18n/locales/en.json | 61 ++++- .../metaforge-web/src/i18n/locales/zh-cn.json | 61 ++++- .../src/model/user/request/user.ts | 7 +- core/metaforge-web/src/router/auth.ts | 6 + core/metaforge-web/src/services/auth.ts | 26 ++ .../src/views/auth/components/AuthCaptcha.vue | 21 ++ .../views/auth/components/AuthCodeField.vue | 19 ++ .../auth/components/AuthIllustration.vue | 43 +++ .../views/auth/components/AuthLanguage.vue | 26 ++ .../src/views/auth/components/AuthShell.vue | 75 ++++++ .../src/views/auth/reset/AuthReset.vue | 71 +++++ .../src/views/auth/signin/AuthSignin.vue | 225 +++++----------- .../src/views/auth/signup/AuthSignup.vue | 254 ++++++------------ 16 files changed, 630 insertions(+), 344 deletions(-) create mode 100644 core/metaforge-web/src/composables/useAuthCaptcha.ts create mode 100644 core/metaforge-web/src/composables/useAuthOptions.ts create mode 100644 core/metaforge-web/src/composables/useEmailCode.ts create mode 100644 core/metaforge-web/src/services/auth.ts create mode 100644 core/metaforge-web/src/views/auth/components/AuthCaptcha.vue create mode 100644 core/metaforge-web/src/views/auth/components/AuthCodeField.vue create mode 100644 core/metaforge-web/src/views/auth/components/AuthIllustration.vue create mode 100644 core/metaforge-web/src/views/auth/components/AuthLanguage.vue create mode 100644 core/metaforge-web/src/views/auth/components/AuthShell.vue create mode 100644 core/metaforge-web/src/views/auth/reset/AuthReset.vue diff --git a/core/metaforge-web/src/composables/useAuthCaptcha.ts b/core/metaforge-web/src/composables/useAuthCaptcha.ts new file mode 100644 index 0000000000..92cdc7a06b --- /dev/null +++ b/core/metaforge-web/src/composables/useAuthCaptcha.ts @@ -0,0 +1,19 @@ +import { ref } from 'vue' +import CaptchaService from '@/services/captcha' + +/** The arithmetic captcha: a new image per timestamp; the answer goes with the form. */ +export function useAuthCaptcha() { + const timestamp = ref(0) + const image = ref('') + const loading = ref(false) + + async function refresh() { + loading.value = true + timestamp.value = Date.now() + const response = await CaptchaService.getCaptcha(timestamp.value) + loading.value = false + image.value = response.status && response.data ? String(response.data.image || '') : '' + } + + return { timestamp, image, loading, refresh } +} diff --git a/core/metaforge-web/src/composables/useAuthOptions.ts b/core/metaforge-web/src/composables/useAuthOptions.ts new file mode 100644 index 0000000000..54f0e4b714 --- /dev/null +++ b/core/metaforge-web/src/composables/useAuthOptions.ts @@ -0,0 +1,21 @@ +import { ref } from 'vue' +import AuthService from '@/services/auth' +import type { AuthOptions } from '@/model/system-setting' + +const options = ref(null) +let pending: Promise | null = null + +/** + * What the sign-in, sign-up and reset pages offer, as administrators set it in system settings. + * Loaded once per visit to the auth pages; the pages render their forms once it is known. + */ +export function useAuthOptions() { + function load() { + pending ||= AuthService.options().then(response => { + if (response.status) options.value = { ...response.data, providers: response.data?.providers || [] } + }).finally(() => { pending = null }) + return pending + } + if (!options.value) load() + return { options, load } +} diff --git a/core/metaforge-web/src/composables/useEmailCode.ts b/core/metaforge-web/src/composables/useEmailCode.ts new file mode 100644 index 0000000000..12ad07407b --- /dev/null +++ b/core/metaforge-web/src/composables/useEmailCode.ts @@ -0,0 +1,39 @@ +import { computed, onScopeDispose, ref } from 'vue' +import { useI18n } from 'vue-i18n' +import { MfMessage } from 'metaforge-ui' +import AuthService from '@/services/auth' +import type { CodePurpose } from '@/model/system-setting' + +/** Mails a code and counts down until another may be asked for. */ +export function useEmailCode(purpose: CodePurpose) { + const { t } = useI18n() + const sending = ref(false) + const seconds = ref(0) + let timer: ReturnType | undefined + + const label = computed(() => seconds.value > 0 ? t('auth.code.resend', { seconds: seconds.value }) : t('auth.code.send')) + const disabled = computed(() => sending.value || seconds.value > 0) + + function countdown(from: number) { + clearInterval(timer) + seconds.value = from + timer = setInterval(() => { + seconds.value-- + if (seconds.value <= 0) clearInterval(timer) + }, 1000) + } + + // The page keeps the button disabled until its email field holds a valid address. + async function send(email: string) { + sending.value = true + const response = await AuthService.sendCode(email.trim(), purpose) + sending.value = false + if (!response.status) { MfMessage.error(response.message); return false } + MfMessage.success(t('auth.code.sent')) + countdown(Number(response.data) || 60) + return true + } + + onScopeDispose(() => clearInterval(timer)) + return { sending, seconds, label, disabled, send } +} diff --git a/core/metaforge-web/src/i18n/locales/en.json b/core/metaforge-web/src/i18n/locales/en.json index c4ff4acf69..0d1e4fbe8b 100644 --- a/core/metaforge-web/src/i18n/locales/en.json +++ b/core/metaforge-web/src/i18n/locales/en.json @@ -1,4 +1,58 @@ { + "auth.brand.feature.convert": "Integrate and convert data", + "auth.brand.feature.plugins": "An extensible plugin ecosystem", + "auth.brand.feature.sources": "Connect many data sources", + "auth.brand.feature.transfer": "Fast import and export", + "auth.brand.illustration": "A database with charts", + "auth.brand.slogan": "The next-generation data processing and integration platform", + "auth.brand.tagline": "Connect · Convert · Manage", + "auth.code.resend": "Resend in {seconds}s", + "auth.code.send": "Get code", + "auth.code.sent": "The code is on its way; check your email", + "auth.field.account": "Username / email", + "auth.field.accountRequired": "Enter your username or email", + "auth.field.captcha": "Answer", + "auth.field.captchaRefresh": "Show another", + "auth.field.captchaRequired": "Enter the answer", + "auth.field.code": "Verification code", + "auth.field.codeRequired": "Enter the code from the email", + "auth.field.codeRule": "The code has 6 digits", + "auth.field.confirm": "Confirm the password", + "auth.field.confirmMismatch": "The passwords do not match", + "auth.field.confirmRequired": "Enter the password again", + "auth.field.email": "Email", + "auth.field.emailInvalid": "Enter a valid email", + "auth.field.emailRequired": "Enter your email", + "auth.field.newPassword": "Set a password", + "auth.field.password": "Password", + "auth.field.passwordRequired": "Enter the password", + "auth.field.passwordRule": "The password has 6–20 characters", + "auth.field.username": "Username", + "auth.field.usernameRequired": "Enter your username", + "auth.field.usernameRule": "3–32 letters, digits or underscores", + "auth.reset.back": "Back to sign in", + "auth.reset.closed": "Password reset is off", + "auth.reset.closedHint": "Ask an administrator to reset your password", + "auth.reset.done": "Password reset; sign in with the new one", + "auth.reset.hint": "Enter your account email, get a code and set a new password.", + "auth.reset.submit": "Reset password", + "auth.reset.title": "Reset password", + "auth.signin.forgot": "Forgot password?", + "auth.signin.remember": "Keep me signed in for {days} days", + "auth.signin.submit": "Sign in", + "auth.signup.agree": "I have read and agree to", + "auth.signup.agreeRequired": "Agree to the terms first", + "auth.signup.and": "and", + "auth.signup.closed": "Sign-up is closed", + "auth.signup.closedHint": "Ask an administrator for an account", + "auth.signup.done": "Signed up; sign in now", + "auth.signup.hasAccount": "Have an account?", + "auth.signup.privacy": "the Privacy Policy", + "auth.signup.submit": "Sign up", + "auth.signup.terms": "the Terms of Service", + "auth.signup.toSignin": "Sign in", + "auth.tab.signin": "Sign in", + "auth.tab.signup": "Sign up", "builder.component.basic": "Basic Component", "builder.component.basic.text": "Text", "builder.component.basic.text.backgroundColor": "Background Color", @@ -2477,17 +2531,10 @@ "template.common.modify": "Modify Template [ $NAME ]", "template.tip.createSuccess": "Template [ $VALUE ] saved successfully", "tools.compare.title": "Data compare", - "user.auth.captchaIsNumberTip": "Captcha must be a number", - "user.auth.captchaSizeTip": "Captcha must be 1-6 characters long", - "user.auth.captchaTip": "Please enter captcha", "user.auth.confirmPasswordTip": "Please enter confirm password", - "user.auth.hasUserTip": "Already have an account?", - "user.auth.notUserTip": "Don’t have an account?", "user.auth.passwordNotMatchTip": "Passwords do not match", "user.auth.passwordSizeTip": "Password must be 6-20 characters long", "user.auth.passwordTip": "Please enter password", - "user.auth.signinTip": "Please enter username and password to sign in", - "user.auth.signupTip": "Please enter username and password to sign up", "user.auth.usernameSizeTip": "Username must be 3-20 characters long", "user.auth.usernameTip": "Please enter username", "user.common.account": "Account", diff --git a/core/metaforge-web/src/i18n/locales/zh-cn.json b/core/metaforge-web/src/i18n/locales/zh-cn.json index cd30ad9847..526cfe73a5 100644 --- a/core/metaforge-web/src/i18n/locales/zh-cn.json +++ b/core/metaforge-web/src/i18n/locales/zh-cn.json @@ -1,4 +1,58 @@ { + "auth.brand.feature.convert": "数据集成与转换", + "auth.brand.feature.plugins": "可扩展的插件生态", + "auth.brand.feature.sources": "多源数据连接", + "auth.brand.feature.transfer": "高效导入与导出", + "auth.brand.illustration": "数据库与图表示意图", + "auth.brand.slogan": "新一代数据处理与集成平台", + "auth.brand.tagline": "数据连接 · 集成转换 · 统一管理", + "auth.code.resend": "{seconds} 秒后重发", + "auth.code.send": "获取验证码", + "auth.code.sent": "验证码已发送，请查收邮件", + "auth.field.account": "用户名 / 邮箱", + "auth.field.accountRequired": "请输入用户名或邮箱", + "auth.field.captcha": "计算结果", + "auth.field.captchaRefresh": "换一张", + "auth.field.captchaRequired": "请输入计算结果", + "auth.field.code": "验证码", + "auth.field.codeRequired": "请输入邮箱验证码", + "auth.field.codeRule": "验证码为 6 位数字", + "auth.field.confirm": "确认密码", + "auth.field.confirmMismatch": "两次输入的密码不一致", + "auth.field.confirmRequired": "请再次输入密码", + "auth.field.email": "邮箱", + "auth.field.emailInvalid": "邮箱格式不正确", + "auth.field.emailRequired": "请输入邮箱", + "auth.field.newPassword": "设置密码", + "auth.field.password": "密码", + "auth.field.passwordRequired": "请输入密码", + "auth.field.passwordRule": "密码长度为 6–20 位", + "auth.field.username": "用户名", + "auth.field.usernameRequired": "请输入用户名", + "auth.field.usernameRule": "3–32 位，支持字母、数字、下划线", + "auth.reset.back": "返回登录", + "auth.reset.closed": "未开启找回密码", + "auth.reset.closedHint": "请联系管理员重置密码", + "auth.reset.done": "密码已重置，请使用新密码登录", + "auth.reset.hint": "输入注册邮箱，获取验证码后设置新密码。", + "auth.reset.submit": "重置密码", + "auth.reset.title": "找回密码", + "auth.signin.forgot": "忘记密码？", + "auth.signin.remember": "{days} 天内保持登录", + "auth.signin.submit": "登录", + "auth.signup.agree": "我已阅读并同意", + "auth.signup.agreeRequired": "请先阅读并同意协议", + "auth.signup.and": "和", + "auth.signup.closed": "暂未开放注册", + "auth.signup.closedHint": "请联系管理员为你创建账号", + "auth.signup.done": "注册成功，请登录", + "auth.signup.hasAccount": "已有账号？", + "auth.signup.privacy": "《隐私政策》", + "auth.signup.submit": "注册", + "auth.signup.terms": "《服务协议》", + "auth.signup.toSignin": "立即登录", + "auth.tab.signin": "登录", + "auth.tab.signup": "注册", "builder.component.basic": "基础组件", "builder.component.basic.text": "文本", "builder.component.basic.text.backgroundColor": "背景颜色", @@ -2477,17 +2531,10 @@ "template.common.modify": "修改模板 [ $NAME ]", "template.tip.createSuccess": "模板 [ $VALUE ] 保存成功", "tools.compare.title": "数据对比", - "user.auth.captchaIsNumberTip": "验证码必须为数字", - "user.auth.captchaSizeTip": "验证码必须在1-6个字符之间", - "user.auth.captchaTip": "请输入验证码", "user.auth.confirmPasswordTip": "请输入确认密码", - "user.auth.hasUserTip": "已有账号?", - "user.auth.notUserTip": "没有账号?", "user.auth.passwordNotMatchTip": "密码不匹配", "user.auth.passwordSizeTip": "密码必须在6-20个字符之间", "user.auth.passwordTip": "请输入密码", - "user.auth.signinTip": "请输入用户名和密码登录", - "user.auth.signupTip": "请输入用户名和密码注册", "user.auth.usernameSizeTip": "用户名必须在3-20个字符之间", "user.auth.usernameTip": "请输入用户名", "user.common.account": "用户", diff --git a/core/metaforge-web/src/model/user/request/user.ts b/core/metaforge-web/src/model/user/request/user.ts index 54b1024f54..623afa3b67 100644 --- a/core/metaforge-web/src/model/user/request/user.ts +++ b/core/metaforge-web/src/model/user/request/user.ts @@ -3,8 +3,13 @@ export interface UserRequest username: string | null | undefined password: string | null | undefined confirmPassword?: string - captcha?: string + captcha?: string | number timestamp?: number + email?: string + /** The mailed code on sign-up. */ + code?: string + /** Keep the sign-in for the remembered days. */ + remember?: boolean // Marks the error message returned after an operation message?: string loading?: boolean diff --git a/core/metaforge-web/src/router/auth.ts b/core/metaforge-web/src/router/auth.ts index 56a680fdef..f6187328ff 100644 --- a/core/metaforge-web/src/router/auth.ts +++ b/core/metaforge-web/src/router/auth.ts @@ -16,6 +16,12 @@ const createAuthRoute = (router: Router) => { path: 'signup', meta: { title: 'user.common.signup' }, component: () => import('@/views/auth/signup/AuthSignup.vue') + }, + { + name: 'reset', + path: 'reset', + meta: { title: 'auth.reset.title' }, + component: () => import('@/views/auth/reset/AuthReset.vue') } ] }) diff --git a/core/metaforge-web/src/services/auth.ts b/core/metaforge-web/src/services/auth.ts new file mode 100644 index 0000000000..e0e757f0db --- /dev/null +++ b/core/metaforge-web/src/services/auth.ts @@ -0,0 +1,26 @@ +import { ResponseModel } from '@/model/response' +import { HttpUtils } from '@/utils/http' +import type { CodePurpose } from '@/model/system-setting' + +const DEFAULT_PATH = '/api/auth' + +/** What the sign-in pages offer, and the email code and password reset flows. */ +class AuthService +{ + options(): Promise + { + return new HttpUtils().get(`${ DEFAULT_PATH }/options`) + } + + sendCode(email: string, purpose: CodePurpose): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/code`, { email, purpose }) + } + + reset(body: { email: string; code: string; password: string }): Promise + { + return new HttpUtils().post(`${ DEFAULT_PATH }/reset`, body) + } +} + +export default new AuthService() diff --git a/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue b/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue new file mode 100644 index 0000000000..47861701a5 --- /dev/null +++ b/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue @@ -0,0 +1,21 @@ + + + diff --git a/core/metaforge-web/src/views/auth/components/AuthCodeField.vue b/core/metaforge-web/src/views/auth/components/AuthCodeField.vue new file mode 100644 index 0000000000..fe433e16d3 --- /dev/null +++ b/core/metaforge-web/src/views/auth/components/AuthCodeField.vue @@ -0,0 +1,19 @@ + + + diff --git a/core/metaforge-web/src/views/auth/components/AuthIllustration.vue b/core/metaforge-web/src/views/auth/components/AuthIllustration.vue new file mode 100644 index 0000000000..1ac2a58eab --- /dev/null +++ b/core/metaforge-web/src/views/auth/components/AuthIllustration.vue @@ -0,0 +1,43 @@ + diff --git a/core/metaforge-web/src/views/auth/components/AuthLanguage.vue b/core/metaforge-web/src/views/auth/components/AuthLanguage.vue new file mode 100644 index 0000000000..ab735d21da --- /dev/null +++ b/core/metaforge-web/src/views/auth/components/AuthLanguage.vue @@ -0,0 +1,26 @@ + + + diff --git a/core/metaforge-web/src/views/auth/components/AuthShell.vue b/core/metaforge-web/src/views/auth/components/AuthShell.vue new file mode 100644 index 0000000000..2551a9f1bd --- /dev/null +++ b/core/metaforge-web/src/views/auth/components/AuthShell.vue @@ -0,0 +1,75 @@ + + + diff --git a/core/metaforge-web/src/views/auth/reset/AuthReset.vue b/core/metaforge-web/src/views/auth/reset/AuthReset.vue new file mode 100644 index 0000000000..a94fb5b99d --- /dev/null +++ b/core/metaforge-web/src/views/auth/reset/AuthReset.vue @@ -0,0 +1,71 @@ + + + diff --git a/core/metaforge-web/src/views/auth/signin/AuthSignin.vue b/core/metaforge-web/src/views/auth/signin/AuthSignin.vue index 3a13c286ce..4af39c38bb 100644 --- a/core/metaforge-web/src/views/auth/signin/AuthSignin.vue +++ b/core/metaforge-web/src/views/auth/signin/AuthSignin.vue @@ -1,190 +1,89 @@ diff --git a/core/metaforge-web/src/views/auth/signup/AuthSignup.vue b/core/metaforge-web/src/views/auth/signup/AuthSignup.vue index c2d0be3215..e67b847663 100644 --- a/core/metaforge-web/src/views/auth/signup/AuthSignup.vue +++ b/core/metaforge-web/src/views/auth/signup/AuthSignup.vue @@ -1,186 +1,108 @@ From e5a56388a7e3c4b483012536d628f7d646081599 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 23:14:05 -0400 Subject: [PATCH 014/232] feat(ui): let MfInput take autofill hints and keep its colors when filled MfInput passes autocomplete (still off by default) and name to its input, so forms can mark usernames, passwords and one-time codes. Browser autofill no longer paints its own background on the bare input inside the field. --- core/metaforge-ui/src/components/MfInput.vue | 8 +++++++- core/metaforge-ui/src/styles/metaforge-ui.css | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/metaforge-ui/src/components/MfInput.vue b/core/metaforge-ui/src/components/MfInput.vue index c318c0dfdf..fe6f2bfcb1 100644 --- a/core/metaforge-ui/src/components/MfInput.vue +++ b/core/metaforge-ui/src/components/MfInput.vue @@ -18,6 +18,9 @@ const props = withDefaults(defineProps<{ readonly?: boolean invalid?: boolean maxlength?: number + /** The browser's autofill hint, e.g. `username`, `current-password`, `new-password`; off by default. */ + autocomplete?: string + name?: string }>(), { modelValue: '', type: 'text', @@ -27,6 +30,8 @@ const props = withDefaults(defineProps<{ disabled: false, readonly: false, invalid: false, + autocomplete: 'off', + name: undefined, }) const emit = defineEmits<{ @@ -81,7 +86,8 @@ defineExpose({ :maxlength="maxlength" class="h-full min-w-0 flex-1 appearance-none border-0 bg-transparent px-3 text-sm text-content placeholder:text-content-placeholder outline-none focus:ring-0" :class="readonly ? 'cursor-default' : ''" - autocomplete="off" + :autocomplete="autocomplete" + :name="name" @input="onInput" @blur="emit('blur', $event)" @keyup.enter="emit('enter', $event)" diff --git a/core/metaforge-ui/src/styles/metaforge-ui.css b/core/metaforge-ui/src/styles/metaforge-ui.css index 1e5be91cd9..0da922aed8 100644 --- a/core/metaforge-ui/src/styles/metaforge-ui.css +++ b/core/metaforge-ui/src/styles/metaforge-ui.css @@ -75,3 +75,16 @@ /* typography */ --font-mf: var(--mf-font-family); } + +/* Browser autofill paints its own (blue) background on the bare , leaving the + * field's icon and padding outside it. Keep the field's own colors instead. */ +@layer components { + .mf-invalidable input:-webkit-autofill, + .mf-invalidable input:-webkit-autofill:hover, + .mf-invalidable input:-webkit-autofill:focus { + -webkit-text-fill-color: var(--mf-color-content); + caret-color: var(--mf-color-content); + box-shadow: 0 0 0 1000px var(--mf-color-surface) inset; + transition: background-color 9999s ease-out 0s; + } +} From 1e5c731d80182aad243068a7a49c22b35e1526e2 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 31 Oct 2026 23:14:05 -0400 Subject: [PATCH 015/232] fix(web): fill the viewport on auth pages and tidy their fields The auth background used min-h-screen, which style.css redefines for the app layout, so the page stopped about 360px short of the bottom. The username rule now sits under the field, the captcha row fills the width, and the captcha comes after the passwords on sign-up with each field carrying its autofill hint, so browsers stop filling the username into it. --- .../src/views/auth/components/AuthCaptcha.vue | 4 ++-- .../views/auth/components/AuthCodeField.vue | 2 +- .../src/views/auth/components/AuthShell.vue | 3 ++- .../src/views/auth/reset/AuthReset.vue | 6 ++--- .../src/views/auth/signin/AuthSignin.vue | 4 ++-- .../src/views/auth/signup/AuthSignup.vue | 23 +++++++++++-------- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue b/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue index 47861701a5..7f933d7cbb 100644 --- a/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue +++ b/core/metaforge-web/src/views/auth/components/AuthCaptcha.vue @@ -1,6 +1,6 @@ @@ -95,9 +136,15 @@ import { computed, onMounted, ref } from 'vue' import { useI18n } from 'vue-i18n' import { MfMessage } from 'metaforge-ui' -import { Save, Send } from '@lucide/vue' +import { Copy, Save, Send } from '@lucide/vue' import SystemSettingService from '@/services/system-setting' -import { defaultAuthSettings, defaultMailSettings, EMAIL_PATTERN, mailReady, URL_PATTERN, waitingForMail } from '@/model/system-setting' +import { apiBaseUrl } from '@/utils/http' +import { + defaultAuthSettings, defaultMailSettings, defaultOAuthGroup, defaultOAuthSettings, EMAIL_PATTERN, incompleteProviders, mailReady, OAUTH_PROVIDERS, + oauthCallbackUrl, oauthConfigured, URL_PATTERN, waitingForMail, +} from '@/model/system-setting' +import type { OAuthProviderId, OAuthSettings } from '@/model/system-setting' +import AuthProviderIcon from '@/views/auth/components/AuthProviderIcon.vue' defineOptions({ name: 'SettingsHome' }) @@ -111,8 +158,41 @@ const testing = ref(false) const testTo = ref('') const auth = ref(defaultAuthSettings()) const mail = ref(defaultMailSettings()) +const oauth = ref(defaultOAuthGroup()) const authRef = ref() const mailRef = ref() +const siteRef = ref() +// Where the browser reaches the server; the callback address uses it while no site address is set. +const origin = apiBaseUrl() + +type OAuthField = 'clientId' | 'clientSecret' | 'name' | 'issuer' | 'authorizationUrl' | 'tokenUrl' | 'userinfoUrl' | 'scopes' +const CLIENT_FIELDS: OAuthField[] = ['clientId', 'clientSecret'] +const OIDC_FIELDS: OAuthField[] = [...CLIENT_FIELDS, 'name', 'issuer', 'authorizationUrl', 'tokenUrl', 'userinfoUrl', 'scopes'] +const fields = (id: OAuthProviderId) => id === 'oidc' ? OIDC_FIELDS : CLIENT_FIELDS +const callback = (id: OAuthProviderId) => oauthCallbackUrl(auth.value.siteUrl, origin, id) + +function placeholder(id: OAuthProviderId, field: OAuthField) { + if (field === 'clientSecret') return oauth.value[id].secretSet ? t('settings.oauth.secretKept') : '' + if (field === 'name') return 'SSO' + if (field.endsWith('Url') || field === 'issuer') return 'https://' + return '' +} + +function status(id: OAuthProviderId) { + const settings = oauth.value[id] + if (!settings.enabled) return { key: 'off', variant: 'default' } as const + return oauthConfigured(id, settings) ? { key: 'on', variant: 'success' } as const : { key: 'incomplete', variant: 'warning' } as const +} + +async function copy(text: string) { + try { + await navigator.clipboard.writeText(text) + MfMessage.success(t('settings.oauth.copied')) + } + catch { + MfMessage.error(t('settings.oauth.copyFailed')) + } +} const ready = computed(() => mailReady(mail.value)) const waiting = computed(() => waitingForMail(auth.value, mail.value)) @@ -123,6 +203,9 @@ const optionalUrl = (value: unknown) => !String(value || '').trim() || URL_PATTE function apply(data: any) { auth.value = { ...defaultAuthSettings(), ...(data?.auth || {}) } mail.value = { ...defaultMailSettings(), ...(data?.mail || {}), password: '' } + const group = defaultOAuthGroup() + for (const id of OAUTH_PROVIDERS) group[id] = { ...defaultOAuthSettings(), ...(data?.oauth?.[id] || {}) as Partial, clientSecret: '' } + oauth.value = group } async function load() { @@ -135,13 +218,18 @@ async function load() { async function save() { try { - await Promise.all([authRef.value?.validate(), mailRef.value?.validate()]) + await Promise.all([authRef.value?.validate(), mailRef.value?.validate(), siteRef.value?.validate()]) } catch { return } + const incomplete = incompleteProviders(oauth.value) + if (incomplete.length) { + MfMessage.error(t('settings.oauth.incompleteSave', { providers: incomplete.map(id => t(`settings.oauth.${id}`)).join('、') })) + return + } saving.value = true - const response = await SystemSettingService.save(auth.value, mail.value) + const response = await SystemSettingService.save(auth.value, mail.value, oauth.value) saving.value = false if (!response.status) { MfMessage.error(response.message); return } apply(response.data) diff --git a/core/metaforge-web/tests/system-setting.test.mjs b/core/metaforge-web/tests/system-setting.test.mjs index 2834fe45c1..b9133026f7 100644 --- a/core/metaforge-web/tests/system-setting.test.mjs +++ b/core/metaforge-web/tests/system-setting.test.mjs @@ -4,7 +4,10 @@ import test from 'node:test' import ts from 'typescript' const source = await readFile(new URL('../src/model/system-setting.ts', import.meta.url), 'utf8') const { outputText } = ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2020 } }) -const { agreements, defaultAuthSettings, defaultMailSettings, EMAIL_PATTERN, mailReady, USERNAME_PATTERN, waitingForMail } = await import(`data:text/javascript;base64,${ Buffer.from(outputText).toString('base64') }`) +const { + agreements, defaultAuthSettings, defaultMailSettings, defaultOAuthGroup, defaultOAuthSettings, EMAIL_PATTERN, incompleteProviders, knownProviders, mailReady, + oauthCallbackUrl, oauthConfigured, oauthError, USERNAME_PATTERN, waitingForMail, +} = await import(`data:text/javascript;base64,${ Buffer.from(outputText).toString('base64') }`) test('mail counts as ready once a host and a sender are set', () => { assert.equal(mailReady(defaultMailSettings()), false) @@ -31,3 +34,36 @@ test('usernames and emails follow the sign-up rules', () => { assert.equal(EMAIL_PATTERN.test('a@example.com'), true) assert.equal(EMAIL_PATTERN.test('a@example'), false) }) + +test('a provider counts as configured once its client, and for OpenID Connect its endpoints, are set', () => { + const client = { ...defaultOAuthSettings(), clientId: 'id', clientSecret: 'secret' } + assert.equal(oauthConfigured('github', defaultOAuthSettings()), false) + assert.equal(oauthConfigured('github', client), true) + // A stored secret counts although the page never receives it. + assert.equal(oauthConfigured('google', { ...client, clientSecret: '', secretSet: true }), true) + assert.equal(oauthConfigured('oidc', client), false) + const endpoints = { authorizationUrl: 'https://sso.example.com/a', tokenUrl: 'https://sso.example.com/t', userinfoUrl: 'https://sso.example.com/u' } + assert.equal(oauthConfigured('oidc', { ...client, ...endpoints }), true) + assert.equal(oauthConfigured('oidc', { ...client, ...endpoints, tokenUrl: 'sso.example.com/t' }), false) +}) + +test('only providers that are on but unfinished block saving', () => { + const group = defaultOAuthGroup() + assert.deepEqual(incompleteProviders(group), []) + group.oidc = { ...group.oidc, enabled: true, clientId: 'id', clientSecret: 'secret' } + group.github = { ...group.github, enabled: true } + assert.deepEqual(incompleteProviders(group), ['github', 'oidc']) +}) + +test('the callback address prefers the site address', () => { + assert.equal(oauthCallbackUrl('', 'http://localhost:9096', 'github'), 'http://localhost:9096/api/auth/oauth/github/callback') + assert.equal(oauthCallbackUrl(' https://mf.example.com/ ', 'http://localhost:9096', 'oidc'), 'https://mf.example.com/api/auth/oauth/oidc/callback') +}) + +test('the sign-in page shows only known providers and known errors', () => { + assert.deepEqual(knownProviders(undefined), []) + assert.deepEqual(knownProviders([{ id: 'github', name: 'GitHub' }, { id: 'gitlab', name: 'GitLab' }, 'oidc', null]), [{ id: 'github', name: 'GitHub' }]) + assert.equal(oauthError('registration'), 'registration') + assert.equal(oauthError(' From ed6488668d6addb8f999b061e396cf0eaee9a60c Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sun, 1 Nov 2026 02:32:18 -0500 Subject: [PATCH 033/232] test(postgresql): verify the plugin against PostgreSQL 15, 16 and 17 PostgreSQLE2ETest creates a schema, table and rows with SQL, lists them as a database, table and columns and reads the rows back. The new connector page lists the verified versions. --- .../reference/connectors/postgresql/home.md | 57 ++++++++++++++ docs/pageforge.yaml | 1 + plugin/metaforge-plugin-postgresql/pom.xml | 6 ++ .../metaforge/plugin/PostgreSQLE2ETest.java | 77 +++++++++++++++++++ 4 files changed, 141 insertions(+) create mode 100644 docs/content/reference/connectors/postgresql/home.md create mode 100644 plugin/metaforge-plugin-postgresql/src/test/java/org/devlive/metaforge/plugin/PostgreSQLE2ETest.java diff --git a/docs/content/reference/connectors/postgresql/home.md b/docs/content/reference/connectors/postgresql/home.md new file mode 100644 index 0000000000..bf8f7fad0b --- /dev/null +++ b/docs/content/reference/connectors/postgresql/home.md @@ -0,0 +1,57 @@ +--- +title: PostgreSQL +--- + + + +#### What is PostgreSQL ? + +PostgreSQL is an open-source relational database known for standards compliance and extensibility. + +#### Configure + +--- + +MetaForge declares the configuration fields inside the plugin class (see `PostgreSQLPlugin#describe`) and connects through the PostgreSQL JDBC driver (`org.postgresql.Driver`). + +::: tabs + === "Configure" + | Field | Required | Default Value | + |:------:|:-------------------:|:-------------:| + | `Name` | :check{20,#3CA34F}: | \- | + | `Host` | :check{20,#3CA34F}: | `127.0.0.1` | + | `Port` | :check{20,#3CA34F}: | `5432` | + === "Authorization" + | Field | Required | Default Value | + |:----------:|:---------------:|:-------------:| + | `Username` | :x{20,#ff0000}: | \- | + | `Password` | :x{20,#ff0000}: | \- | + === "Advanced" + | Field | Required | Default Value | + |:----------:|:---------------:|:-------------:| + | `Database` | :x{20,#ff0000}: | \- | +::: + +#### Version (Validation) + +--- + +!!! note + + The plugin's e2e test `PostgreSQLE2ETest` verifies each version below in a container: the version check, then a schema, table and columns created with SQL are listed as a database, table and columns, and their rows read back. + + - [x] 15 + - [x] 16 + - [x] 17 + +!!! + +#### SQL and concepts + +--- + +| MetaForge | PostgreSQL | +|:--|:--| +| Database | Schema of the configured database | +| Table | Table or view | +| Column | Column | diff --git a/docs/pageforge.yaml b/docs/pageforge.yaml index c6c15394ab..cae56f9489 100644 --- a/docs/pageforge.yaml +++ b/docs/pageforge.yaml @@ -221,6 +221,7 @@ nav: - /reference/connectors/clickhouse/home.md - /reference/connectors/cratedb/home.md - /reference/connectors/mysql/home.md + - /reference/connectors/postgresql/home.md - /reference/connectors/redis/home.md - /reference/connectors/mongocommunity/home.md - /reference/connectors/h2/home.md diff --git a/plugin/metaforge-plugin-postgresql/pom.xml b/plugin/metaforge-plugin-postgresql/pom.xml index 14b634e7ba..4ec77b8b0d 100644 --- a/plugin/metaforge-plugin-postgresql/pom.xml +++ b/plugin/metaforge-plugin-postgresql/pom.xml @@ -29,6 +29,12 @@ postgresql ${pgsql.version} + + org.devlive.metaforge + metaforge-test-e2e + ${project.version} + test + diff --git a/plugin/metaforge-plugin-postgresql/src/test/java/org/devlive/metaforge/plugin/PostgreSQLE2ETest.java b/plugin/metaforge-plugin-postgresql/src/test/java/org/devlive/metaforge/plugin/PostgreSQLE2ETest.java new file mode 100644 index 0000000000..f680cc4e7d --- /dev/null +++ b/plugin/metaforge-plugin-postgresql/src/test/java/org/devlive/metaforge/plugin/PostgreSQLE2ETest.java @@ -0,0 +1,77 @@ +package org.devlive.metaforge.plugin; + +import org.devlive.metaforge.spi.PluginService; +import org.devlive.metaforge.spi.model.RuntimeConfigure; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; +import java.util.List; + +import static org.devlive.metaforge.test.e2e.PluginE2E.assumeDocker; +import static org.devlive.metaforge.test.e2e.PluginE2E.column; +import static org.devlive.metaforge.test.e2e.PluginE2E.configure; +import static org.devlive.metaforge.test.e2e.PluginE2E.first; +import static org.devlive.metaforge.test.e2e.PluginE2E.ok; + +/** + * Runs the PostgreSQL plugin through its JDBC driver against every PostgreSQL release line it + * documents as verified: schemas, tables and columns created with SQL are listed as databases, + * tables and columns, and their rows read back. + */ +@RunWith(Parameterized.class) +public class PostgreSQLE2ETest +{ + @Parameterized.Parameters(name = "PostgreSQL {0}") + public static List versions() + { + return List.of("15", "16", "17"); + } + + private final String version; + + public PostgreSQLE2ETest(String version) + { + this.version = version; + } + + @BeforeClass + public static void docker() + { + assumeDocker(); + } + + @Test + public void schemasTablesAndColumnsAreListedAndRead() + { + try (GenericContainer container = new GenericContainer<>("postgres:" + version) + .withEnv("POSTGRES_PASSWORD", "metaforge") + .withExposedPorts(5432) + .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*", 2).withStartupTimeout(Duration.ofMinutes(3)))) { + container.start(); + PluginService service = new PostgreSQLService(); + RuntimeConfigure configure = configure(container.getHost(), container.getMappedPort(5432), "postgres", "metaforge", "postgres"); + + String reported = first(ok("validator", service.execute(configure, service.validator()))); + Assert.assertTrue(reported + " is not " + version, reported.startsWith("PostgreSQL " + version + ".")); + + ok("create schema", service.execute(configure, "CREATE SCHEMA e2e")); + Assert.assertTrue(column(ok("databases", service.getDatabases(configure)), "object_name").contains("e2e")); + ok("create table", service.execute(configure, "CREATE TABLE e2e.items (id INT PRIMARY KEY, name VARCHAR(32) NOT NULL)")); + ok("insert", service.execute(configure, "INSERT INTO e2e.items VALUES (1, 'a'), (2, 'b')")); + Assert.assertTrue(column(ok("tables", service.getTables(configure, "e2e")), "object_name").contains("items")); + List columns = column(ok("columns", service.getColumns(configure, "e2e", "items")), "object_name"); + Assert.assertTrue(columns.toString(), columns.containsAll(List.of("id", "name"))); + Assert.assertEquals("b", first(ok("select", service.execute(configure, "SELECT name FROM e2e.items WHERE id = 2")))); + Assert.assertEquals("2", first(ok("count", service.execute(configure, "SELECT COUNT(*) AS total FROM e2e.items")))); + + ok("drop schema", service.execute(configure, "DROP SCHEMA e2e CASCADE")); + Assert.assertFalse(column(ok("databases", service.getDatabases(configure)), "object_name").contains("e2e")); + } + } +} From 9ca4b2dc16458ae05deaea4d321639e8e50bc10c Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sun, 1 Nov 2026 02:34:44 -0500 Subject: [PATCH 034/232] fix(ui): keep dropdowns with a disabled trigger closed A disabled button lets clicks through to the dropdown wrapper, so its menu still opened. MfDropdown does not open while its trigger button is disabled or when it is disabled itself. --- core/metaforge-ui/src/components/MfDropdown.vue | 13 +++++++++++++ core/metaforge-ui/tests/popup-interaction.test.mjs | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/metaforge-ui/src/components/MfDropdown.vue b/core/metaforge-ui/src/components/MfDropdown.vue index d6cc5952d2..a8bdbb90c5 100644 --- a/core/metaforge-ui/src/components/MfDropdown.vue +++ b/core/metaforge-ui/src/components/MfDropdown.vue @@ -6,9 +6,12 @@ import { useOverlayPosition, type OverlayPlacement } from '../composables/useOve const props = withDefaults(defineProps<{ trigger?: ('click' | 'hover' | 'contextmenu')[] | string placement?: OverlayPlacement + /** Never opens. A disabled button as the trigger has the same effect. */ + disabled?: boolean }>(), { trigger: 'hover', placement: 'bottomLeft', + disabled: false, }) const open = defineModel('open', { default: false }) @@ -35,7 +38,13 @@ function onOverlayClick(event: Event): void { if (target?.closest?.('button, a, [role="menuitem"]')) close() } +/** A disabled trigger lets clicks through (pointer-events: none) to this wrapper, so it is checked here. */ +function blocked(): boolean { + return props.disabled || !!root.value?.querySelector?.(':scope > :is(button, [role="button"]):is(:disabled, [aria-disabled="true"])') +} + function show(): void { + if (blocked()) return if (timer) clearTimeout(timer) timer = undefined open.value = true @@ -52,6 +61,10 @@ function toggle(event: Event): void { if (triggers.value.includes('contextmenu')) { event.preventDefault() } + if (blocked()) { + open.value = false + return + } open.value = !open.value } diff --git a/core/metaforge-ui/tests/popup-interaction.test.mjs b/core/metaforge-ui/tests/popup-interaction.test.mjs index 8a81777b71..44792553e0 100644 --- a/core/metaforge-ui/tests/popup-interaction.test.mjs +++ b/core/metaforge-ui/tests/popup-interaction.test.mjs @@ -31,6 +31,18 @@ test('click dropdown closes outside and unregisters on unmount', async () => { assert.equal(host.document.count('keydown'), 0) }) +test('a disabled dropdown does not open from its trigger', async () => { + const host = mountHost(h(Dropdown, { trigger: 'click', disabled: true }, { + default: () => h('button', 'Open'), + overlay: () => h('div', { 'data-panel': 'dropdown' }, 'Choices'), + })) + try { + await click(host.document, host.find(node => node.tag === 'button')) + assert.ok(!host.find(node => node.props['data-panel'] === 'dropdown')) + } + finally { host.unmount() } +}) + test('an outside control that stops click propagation still dismisses the dropdown and acts once', async () => { let actions = 0 const host = mountHost(h('div', [ From 102ce1e54c196fb911845a2f63097fac224abfad Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sun, 1 Nov 2026 02:34:44 -0500 Subject: [PATCH 035/232] fix(query): stop squeezing side panel search fields and lock dataset menu The side panel body is a scrolling column, so long lists shrank its search fields; its parts keep their height now. The apply-to-dataset menu stays closed while a dataset can not be created. --- .../src/views/pages/admin/query/QueryResultPanel.vue | 2 +- .../src/views/pages/admin/query/QuerySidePanel.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/metaforge-web/src/views/pages/admin/query/QueryResultPanel.vue b/core/metaforge-web/src/views/pages/admin/query/QueryResultPanel.vue index 6a34b6bd36..9d54100fb1 100644 --- a/core/metaforge-web/src/views/pages/admin/query/QueryResultPanel.vue +++ b/core/metaforge-web/src/views/pages/admin/query/QueryResultPanel.vue @@ -25,7 +25,7 @@ {{ $t('dataset.common.visual') }} {{ $t('dataset.common.create') }} - ··· + ··· diff --git a/core/metaforge-web/src/views/pages/admin/query/QuerySidePanel.vue b/core/metaforge-web/src/views/pages/admin/query/QuerySidePanel.vue index 425c9b7a88..c2a90f7b14 100644 --- a/core/metaforge-web/src/views/pages/admin/query/QuerySidePanel.vue +++ b/core/metaforge-web/src/views/pages/admin/query/QuerySidePanel.vue @@ -14,7 +14,7 @@ -
+
diff --git a/core/metaforge-web/src/views/pages/store/components/HistoryView.vue b/core/metaforge-web/src/views/pages/store/components/HistoryView.vue new file mode 100644 index 0000000000..63fed6ade6 --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/HistoryView.vue @@ -0,0 +1,53 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/InstalledView.vue b/core/metaforge-web/src/views/pages/store/components/InstalledView.vue new file mode 100644 index 0000000000..998517e32d --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/InstalledView.vue @@ -0,0 +1,135 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/MarketView.vue b/core/metaforge-web/src/views/pages/store/components/MarketView.vue new file mode 100644 index 0000000000..b76c820757 --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/MarketView.vue @@ -0,0 +1,203 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/PluginCard.vue b/core/metaforge-web/src/views/pages/store/components/PluginCard.vue new file mode 100644 index 0000000000..23ae8f9a3b --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/PluginCard.vue @@ -0,0 +1,76 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/PluginDetailDrawer.vue b/core/metaforge-web/src/views/pages/store/components/PluginDetailDrawer.vue new file mode 100644 index 0000000000..856fcdbe85 --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/PluginDetailDrawer.vue @@ -0,0 +1,234 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/PluginInfo.vue b/core/metaforge-web/src/views/pages/store/components/PluginInfo.vue deleted file mode 100644 index 94b90e1abf..0000000000 --- a/core/metaforge-web/src/views/pages/store/components/PluginInfo.vue +++ /dev/null @@ -1,114 +0,0 @@ - - - diff --git a/core/metaforge-web/src/views/pages/store/components/PluginSymbol.vue b/core/metaforge-web/src/views/pages/store/components/PluginSymbol.vue new file mode 100644 index 0000000000..ea61a7006f --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/PluginSymbol.vue @@ -0,0 +1,65 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/StoreDialogs.vue b/core/metaforge-web/src/views/pages/store/components/StoreDialogs.vue new file mode 100644 index 0000000000..aa26226e2f --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/StoreDialogs.vue @@ -0,0 +1,230 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/StoreSidebar.vue b/core/metaforge-web/src/views/pages/store/components/StoreSidebar.vue new file mode 100644 index 0000000000..6750ced81c --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/StoreSidebar.vue @@ -0,0 +1,91 @@ + + + diff --git a/core/metaforge-web/src/views/pages/store/components/UpdatesView.vue b/core/metaforge-web/src/views/pages/store/components/UpdatesView.vue new file mode 100644 index 0000000000..62bccd1a23 --- /dev/null +++ b/core/metaforge-web/src/views/pages/store/components/UpdatesView.vue @@ -0,0 +1,76 @@ + + + diff --git a/core/metaforge-web/tests/plugin-store.test.mjs b/core/metaforge-web/tests/plugin-store.test.mjs new file mode 100644 index 0000000000..a730c20527 --- /dev/null +++ b/core/metaforge-web/tests/plugin-store.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { sourceModule } from '../../metaforge-ui/tests/component.mjs' + +const { formatPackageSize, hasNewerPluginVersion, mergeStorePlugins, readStoreCatalogue } = await sourceModule('../../metaforge-web/src/model/store.ts') + +test('only a newer published version is offered as an update', () => { + assert.equal(hasNewerPluginVersion('2026.0.0', '2025.10.0'), false) + assert.equal(hasNewerPluginVersion('2026.0.0', '2026.0.0-SNAPSHOT'), false) + assert.equal(hasNewerPluginVersion('2026.0.0-SNAPSHOT', '2026.0.0'), true) + assert.equal(hasNewerPluginVersion('2.9.0', '2.10.0'), true) + assert.equal(hasNewerPluginVersion('', '2.10.0'), false) +}) + +test('catalogue and runtime aliases merge once and only server references are shown', () => { + const catalogue = readStoreCatalogue({ children: [ + { key: 'metaforge-plugin-mysql', label: 'MySQL', type: 'Connector', version: '2026.0.0', url: 'https://example.test/plugin.tar.gz', refs: [{ name: 'demo' }] }, + { key: 'metaforge-convert-json', label: 'JSON', type: 'Converter' }, + ] }) + const rows = mergeStorePlugins(catalogue, [ + { name: 'MySQL', key: 'mysql', type: 'CONNECTOR', version: '2025.0.0', state: 'STOPPED' }, + { name: 'Redis', key: 'redis', type: 'CONNECTOR', version: '2026.0.0', state: 'CREATED' }, + ], { MySQL: [{ kind: 'source', name: 'Warehouse' }] }) + assert.deepEqual(rows.map(row => row.type), ['Connector', 'Convert', 'Connector']) + assert.equal(rows[0].registeredName, 'MySQL') + assert.equal(rows[0].hasUpdate, true) + assert.equal(rows[0].enabled, false) + assert.deepEqual(rows[0].refs, [{ kind: 'source', name: 'Warehouse' }]) + assert.equal(rows[2].refs, undefined) +}) + +test('built catalogues carry size, checksum, sources and exact download counts', () => { + const sha = 'a'.repeat(64) + const catalogue = readStoreCatalogue({ children: [ + { key: 'metaforge-plugin-mysql', label: 'MySQL', version: '2026.0.0', size: 26568847, sha256: sha, requires: '2026.0.0', + features: ['query', 'export'], sources: { maven: 'https://repo1.maven.org/x.tar.gz', cdn: 'https://cdn.example/x.tar.gz', evil: 'javascript:alert(1)' } }, + { key: 'metaforge-plugin-old', label: 'Old', size: '12 MB', sha256: 'not-a-sha' }, + ] }) + assert.equal(catalogue[0].sha256, sha) + assert.deepEqual(Object.keys(catalogue[0].sources), ['maven', 'cdn']) + assert.equal(catalogue[1].sha256, undefined) + assert.equal(formatPackageSize(catalogue[0].size), '25.3 MB') + assert.equal(formatPackageSize(catalogue[1].size), '12 MB') + assert.equal(formatPackageSize(undefined), '--') + assert.equal(formatPackageSize(512), '512 B') + const rows = mergeStorePlugins(catalogue, [], {}, { 'metaforge-plugin-mysql': '9007199254740993' }) + assert.equal(rows[0].downloads, '9007199254740993') + assert.equal(rows[1].downloads, undefined, 'unknown counts stay unknown, never zero') +}) + +test('installed plugins describe themselves when the catalogue lacks details', () => { + const catalogue = readStoreCatalogue({ children: [{ key: 'metaforge-plugin-mysql', label: 'MySQL', description: 'From catalogue' }] }) + const rows = mergeStorePlugins(catalogue, [ + { name: 'MySql', key: 'metaforge-plugin-mysql', store: { label: 'Ignored', description: 'Ignored', logo: 'mysql.svg', features: ['query'] } }, + { name: 'Custom', key: 'acme-plugin-x', store: { label: 'Acme X', description: 'Self built', type: 'Connector', features: ['export'] } }, + ], {}) + assert.equal(rows[0].label, 'MySQL', 'catalogue values win') + assert.equal(rows[0].description, 'From catalogue') + assert.equal(rows[0].logo, 'mysql.svg', 'missing catalogue fields come from the plugin') + assert.deepEqual(rows[0].features, ['query']) + assert.equal(rows[1].label, 'Acme X') + assert.equal(rows[1].description, 'Self built') + assert.equal(rows[1].type, 'Connector') + assert.deepEqual(rows[1].features, ['export']) +}) From 9525eb972f499cc072fcc513f8e9547c1bb6a615 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Wed, 4 Nov 2026 10:17:42 -0500 Subject: [PATCH 139/232] feat(report): share reports through public links and iframe embeds The owner turns on public access per report and gets a random 192-bit link, which can be rotated at any time. Visitors read the saved definition only: the query chart snapshot, its read-only refresh, or the saved dataset query, run as the owner with a 30 second cache. They never send SQL and never see the SQL, data source, owner or internal errors. - /embed/report/{token} serves the web UI with CSP frame-ancestors from the owner's allowed sites (any site when empty) and no X-Frame-Options; every other page stays same-origin - saving a report never changes its sharing; new reports start private - reports from the earlier chart editor cannot be shared, they ran SQL in the browser --- .../metaforge/security/SecurityConfigure.java | 16 +- .../controller/PublicReportController.java | 48 +++ .../server/controller/ReportController.java | 14 +- .../SharedReportPageController.java | 69 +++++ .../changelog/core/core-0012-report-share.xml | 21 ++ .../db/changelog/db.changelog-master.xml | 1 + .../service/entity/ReportEntity.java | 19 ++ .../service/repository/ReportRepository.java | 4 + .../service/service/ReportShareService.java | 71 +++++ .../service/impl/ReportServiceImpl.java | 11 +- .../service/impl/ReportShareServiceImpl.java | 290 ++++++++++++++++++ .../impl/ReportShareServiceImplTest.java | 251 +++++++++++++++ 12 files changed, 809 insertions(+), 6 deletions(-) create mode 100644 core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java create mode 100644 core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java create mode 100644 core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java create mode 100644 core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java create mode 100644 core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java diff --git a/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java index 0e221280ff..6069217ffd 100644 --- a/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java +++ b/core/metaforge-security/src/main/java/org/devlive/metaforge/security/SecurityConfigure.java @@ -13,6 +13,10 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; +import org.springframework.security.web.header.writers.DelegatingRequestMatcherHeaderWriter; +import org.springframework.security.web.header.writers.frameoptions.XFrameOptionsHeaderWriter; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; +import org.springframework.security.web.util.matcher.NegatedRequestMatcher; @Configuration @EnableWebSecurity @@ -69,14 +73,20 @@ protected void configure(HttpSecurity http) .exceptionHandling().authenticationEntryPoint(unauthorizedHandler).and() .sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS).and() .authorizeRequests() - .antMatchers("/", "/**/*.js", "/**/*.css", "/assets/**", "/api/auth/**", "/fonts/**", "/static/**", "/h2-console/**", "/api/v1/captcha", "/api/v1/table/dataDownload/**", "/upload/**", "/api/v1/setup/**") + .antMatchers("/", "/**/*.js", "/**/*.css", "/assets/**", "/api/auth/**", "/fonts/**", "/static/**", "/h2-console/**", "/api/v1/captcha", "/api/v1/table/dataDownload/**", "/upload/**", "/api/v1/setup/**", + // Shared reports: token-addressed public API and the pages that render it + "/api/v1/public/**", "/share/**", "/embed/**") .permitAll() .requestMatchers(new AdminApiRequestMatcher()) .hasAuthority("ADMIN") .anyRequest() .authenticated(); - // Fixed load denied by X-Frame-Options - http.headers().frameOptions().sameOrigin(); + // Same-origin framing everywhere except embedded report pages, whose CSP frame-ancestors + // carries the sites the report owner allowed. + http.headers().frameOptions().disable() + .addHeaderWriter(new DelegatingRequestMatcherHeaderWriter( + new NegatedRequestMatcher(new AntPathRequestMatcher("/embed/**")), + new XFrameOptionsHeaderWriter(XFrameOptionsHeaderWriter.XFrameOptionsMode.SAMEORIGIN))); http.addFilterBefore(authenticationJwtTokenFilter(), UsernamePasswordAuthenticationFilter.class); } } diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java new file mode 100644 index 0000000000..57b9b70272 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/PublicReportController.java @@ -0,0 +1,48 @@ +package org.devlive.metaforge.server.controller; + +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +/** + * Anonymous access to shared reports by their share token. Only the saved report definition is + * evaluated; visitors cannot send SQL, queries or report codes. + */ +@RestController +@RequestMapping(value = "/api/v1/public/report") +public class PublicReportController +{ + private final ReportShareService service; + + public PublicReportController(ReportShareService service) + { + this.service = service; + } + + @GetMapping("{token}") + public ResponseEntity> view(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()).body(service.view(token)); + } + + @GetMapping("{token}/refresh") + public ResponseEntity>>> refresh(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.maxAge(Duration.ofSeconds(30))).body(service.refresh(token)); + } + + @GetMapping("{token}/data") + public ResponseEntity> data(@PathVariable("token") String token) + { + return ResponseEntity.ok().cacheControl(CacheControl.maxAge(Duration.ofSeconds(30))).body(service.data(token)); + } +} diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java index becc875223..23abeb1620 100644 --- a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/ReportController.java @@ -4,10 +4,13 @@ import org.devlive.metaforge.service.entity.ReportEntity; import org.devlive.metaforge.service.repository.ReportRepository; import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; import org.springframework.http.CacheControl; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -20,11 +23,20 @@ public class ReportController extends BaseController { private final ReportService service; + private final ReportShareService shareService; - protected ReportController(ReportRepository repository, ReportService service) + protected ReportController(ReportRepository repository, ReportService service, ReportShareService shareService) { super(repository, service); this.service = service; + this.shareService = shareService; + } + + // Owner only: public link, iframe embedding, allowed sites and link rotation + @PutMapping("{code}/share") + public CommonResponse share(@PathVariable("code") String code, @RequestBody ReportShareService.ShareRequest request) + { + return shareService.share(code, request); } @GetMapping("{code}/chart-refresh") diff --git a/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java new file mode 100644 index 0000000000..9427028fe9 --- /dev/null +++ b/core/metaforge-server/src/main/java/org/devlive/metaforge/server/controller/SharedReportPageController.java @@ -0,0 +1,69 @@ +package org.devlive.metaforge.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.core.io.ClassPathResource; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * Serves the web UI document for shared reports on real paths, so each response can carry its own + * framing policy: /share/report/{token} stays same-origin only, /embed/report/{token} may be framed + * by the sites the owner allowed (CSP frame-ancestors). The web app routes both to the public view. + */ +@Slf4j +@Controller +public class SharedReportPageController +{ + private static final String[] INDEX = {"resources/index.html", "static/index.html", "public/index.html"}; + + private final ReportShareService service; + + public SharedReportPageController(ReportShareService service) + { + this.service = service; + } + + @GetMapping("/share/report/{token}") + public ResponseEntity share(@PathVariable("token") String token) + { + return page(HttpStatus.OK, "frame-ancestors 'self'"); + } + + @GetMapping("/embed/report/{token}") + public ResponseEntity embed(@PathVariable("token") String token) + { + // Unknown, private or non-embeddable reports render the "not available" view and refuse framing. + return page(HttpStatus.OK, "frame-ancestors " + service.frameAncestors(token).orElse("'none'")); + } + + private ResponseEntity page(HttpStatus status, String frameAncestors) + { + for (String location : INDEX) { + ClassPathResource resource = new ClassPathResource(location); + if (!resource.exists()) { + continue; + } + try (InputStream stream = resource.getInputStream()) { + return ResponseEntity.status(status) + .contentType(new MediaType(MediaType.TEXT_HTML, StandardCharsets.UTF_8)) + .cacheControl(CacheControl.noStore()) + .header("Content-Security-Policy", frameAncestors) + .body(new String(stream.readAllBytes(), StandardCharsets.UTF_8)); + } + catch (IOException e) { + log.warn("Cannot read the web UI document {}", location, e); + } + } + return ResponseEntity.status(HttpStatus.NOT_FOUND).contentType(MediaType.TEXT_PLAIN).body("The web UI is not packaged with this server"); + } +} diff --git a/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml b/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml new file mode 100644 index 0000000000..b37e3543b7 --- /dev/null +++ b/core/metaforge-server/src/main/resources/db/changelog/core/core-0012-report-share.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + diff --git a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml index 2f2dc4488e..f477c1804f 100644 --- a/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml +++ b/core/metaforge-server/src/main/resources/db/changelog/db.changelog-master.xml @@ -37,4 +37,5 @@ + diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java index ab717e1824..0aebefd338 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/entity/ReportEntity.java @@ -68,6 +68,25 @@ public class ReportEntity @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) private Long datasetId; + // Public link and iframe embedding. The token is the only public identifier of a shared report; + // only its owner reads these fields and they change through the share endpoint, never a save. + @Column("share_token") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String shareToken; + + @Column("share_enabled") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareEnabled; + + @Column("share_embed") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private boolean shareEmbed; + + // Origins allowed to frame the report, one per line; empty allows any site. + @Column("share_origins") + @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) + private String shareOrigins; + @Transient @Reference("datasetId") @JsonView(value = {EntityView.UserView.class, EntityView.AdminView.class}) diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java index 33e893e47d..638eebfe30 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/repository/ReportRepository.java @@ -6,6 +6,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import java.util.Optional; + public interface ReportRepository extends BaseRepository { @@ -16,4 +18,6 @@ default Visibility visibleTo(UserEntity user) } Page findAllByUserId(Long userId, Pageable pageable); + + Optional findByShareToken(String shareToken); } diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java new file mode 100644 index 0000000000..799c2d84ec --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/ReportShareService.java @@ -0,0 +1,71 @@ +package org.devlive.metaforge.service.service; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.devlive.metaforge.common.response.CommonResponse; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Public links and iframe embedding of reports. The share token is the only public identifier; + * public reads run the report's saved definition as its owner and never accept SQL or a query + * from the visitor. + */ +public interface ReportShareService +{ + /** Owner only: turn sharing on or off, allow embedding, restrict framing origins, rotate the link. */ + CommonResponse share(String code, ShareRequest request); + + /** The public view of a shared report, without its SQL, data source or owner. */ + CommonResponse view(String token); + + /** Fresh rows of a shared dynamic query chart. */ + CommonResponse>> refresh(String token); + + /** Rows of a shared dataset report, computed from the saved dataset query. */ + CommonResponse data(String token); + + /** + * The CSP frame-ancestors value for an embeddable shared report, or empty when the report + * cannot be embedded. + */ + Optional frameAncestors(String token); + + @Data + @NoArgsConstructor + @AllArgsConstructor + class ShareRequest + { + private boolean enabled; + private boolean embed; + private String origins; + private boolean rotate; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + class ShareState + { + private boolean enabled; + private boolean embed; + private String origins; + private String token; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + class PublicReport + { + private String name; + private String description; + private String type; + private boolean realtime; + private String configure; + private boolean embed; + } +} diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java index 0bca78f3e6..b6b8c35b28 100644 --- a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportServiceImpl.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; @Service @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") @@ -58,10 +59,16 @@ public CommonResponse> getAll(FilterBody filter) public CommonResponse saveOrUpdate(BaseRepository repository, ReportEntity configure) { var user = UserDetailsService.getUser(); - if (user.getId() == null || (configure.getId() != null && this.repository.findById(configure.getId()) - .filter(existing -> Objects.equals(existing.getUserId(), user.getId())).isEmpty())) { + Optional existing = configure.getId() == null ? Optional.empty() : this.repository.findById(configure.getId()) + .filter(value -> Objects.equals(value.getUserId(), user.getId())); + if (user.getId() == null || (configure.getId() != null && existing.isEmpty())) { return CommonResponse.failure("Report not found"); } + // Sharing changes only through the share endpoint: a save keeps the stored values, and a new report starts private. + configure.setShareToken(existing.map(ReportEntity::getShareToken).orElse(null)); + configure.setShareEnabled(existing.map(ReportEntity::isShareEnabled).orElse(false)); + configure.setShareEmbed(existing.map(ReportEntity::isShareEmbed).orElse(false)); + configure.setShareOrigins(existing.map(ReportEntity::getShareOrigins).orElse(null)); JsonNode chart = chartRefresh.chart(configure); if (chart != null && configure.getSource() == null) { return CommonResponse.failure("Dynamic chart data source not found"); diff --git a/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java new file mode 100644 index 0000000000..ebb7a93e9b --- /dev/null +++ b/core/metaforge-service/src/main/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImpl.java @@ -0,0 +1,290 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.body.adhoc.Adhoc; +import org.devlive.metaforge.service.entity.DataSetEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.ReportType; +import org.devlive.metaforge.service.repository.DataSetRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DataSetService; +import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Base64; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +@Slf4j +@Service +@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "Collaborators are Spring-managed singletons") +public class ReportShareServiceImpl + implements ReportShareService +{ + static final Pattern TOKEN = Pattern.compile("^[A-Za-z0-9_-]{32}$"); + private static final Pattern ORIGIN = Pattern.compile("^https?://(\\*\\.)?[a-z0-9-]+(\\.[a-z0-9-]+)*(:\\d{1,5})?$"); + private static final int MAX_ORIGINS = 20; + // Public traffic must not turn into one database query per page view. + private static final long CACHE_MILLIS = 30_000; + private static final int CACHE_SIZE = 1000; + + private final ReportRepository repository; + private final UserRepository userRepository; + private final DataSetRepository dataSetRepository; + private final ReportService reportService; + private final DataSetService dataSetService; + private final ObjectMapper mapper; + private final SecureRandom random = new SecureRandom(); + private final Map cache = new ConcurrentHashMap<>(); + + public ReportShareServiceImpl(ReportRepository repository, UserRepository userRepository, DataSetRepository dataSetRepository, + ReportService reportService, DataSetService dataSetService, ObjectMapper mapper) + { + this.repository = repository; + this.userRepository = userRepository; + this.dataSetRepository = dataSetRepository; + this.reportService = reportService; + this.dataSetService = dataSetService; + this.mapper = mapper; + } + + @Override + public CommonResponse share(String code, ShareRequest request) + { + Long userId = UserDetailsService.getUser().getId(); + ReportEntity report = userId == null ? null : repository.findByCode(code) + .filter(value -> Objects.equals(value.getUserId(), userId)) + .orElse(null); + if (report == null) { + return CommonResponse.failure("Report not found"); + } + if (request.isEnabled() && !shareable(report)) { + return CommonResponse.failure("This report was saved by the earlier chart editor. Save it again from the query page to share it."); + } + Optional origins = normalizeOrigins(request.getOrigins()); + if (origins.isEmpty()) { + return CommonResponse.failure("Allowed sites must be origins such as https://example.com, at most " + MAX_ORIGINS); + } + if (request.isRotate() || report.getShareToken() == null) { + report.setShareToken(newToken()); + } + report.setShareEnabled(request.isEnabled()); + report.setShareEmbed(request.isEmbed()); + report.setShareOrigins(origins.get().isEmpty() ? null : origins.get()); + repository.save(report); + evict(report.getShareToken()); + return CommonResponse.success(state(report)); + } + + @Override + public CommonResponse view(String token) + { + return shared(token) + .map(report -> CommonResponse.success(new PublicReport(report.getName(), report.getDescription(), + report.getType() == null ? null : report.getType().name(), report.isRealtime(), report.getConfigure(), report.isShareEmbed()))) + .orElseGet(() -> CommonResponse.failure("Report not found")); + } + + @Override + public CommonResponse>> refresh(String token) + { + Optional report = shared(token).filter(value -> value.getType() == ReportType.QUERY && value.isRealtime()); + if (report.isEmpty()) { + return CommonResponse.failure("Dynamic chart not found"); + } + // The owner's own refresh path: read-only transaction, row limit and SQL checks included. + return cached("refresh:" + token, () -> publicly(() -> asOwner(report.get(), () -> reportService.refreshChart(report.get().getCode())))); + } + + @Override + public CommonResponse data(String token) + { + Optional report = shared(token).filter(value -> value.getType() == ReportType.DATASET); + if (report.isEmpty() || report.get().getDatasetId() == null) { + return CommonResponse.failure("Report not found"); + } + Optional dataset; + try { + dataset = dataSetRepository.findById(report.get().getDatasetId()); + } + catch (RuntimeException e) { + log.warn("Shared report dataset lookup failed: {}", e.getClass().getSimpleName()); + return CommonResponse.failure("Report not found"); + } + if (dataset.isEmpty()) { + return CommonResponse.failure("Report not found"); + } + Adhoc query; + try { + query = mapper.readValue(report.get().getQuery(), Adhoc.class); + } + catch (JsonProcessingException e) { + return CommonResponse.failure("Report query is invalid"); + } + return cached("data:" + token, () -> publicly(() -> { + CommonResponse response = asOwner(report.get(), () -> dataSetService.adhoc(dataset.get().getCode(), query)); + return Boolean.TRUE.equals(response.getStatus()) ? CommonResponse.success(response.getData()) : CommonResponse.failure(String.valueOf(response.getMessage())); + })); + } + + @Override + public Optional frameAncestors(String token) + { + return shared(token) + .filter(ReportEntity::isShareEmbed) + .map(report -> StringUtils.isBlank(report.getShareOrigins()) ? "*" + : "'self' " + String.join(" ", report.getShareOrigins().split("\n"))); + } + + Optional shared(String token) + { + if (token == null || !TOKEN.matcher(token).matches()) { + return Optional.empty(); + } + try { + return repository.findByShareToken(token) + .filter(report -> report.isActive() && report.isShareEnabled() && token.equals(report.getShareToken()) && shareable(report)); + } + catch (RuntimeException e) { + // Anonymous callers never see internal errors such as SQL text; to them the report is unavailable. + log.warn("Shared report lookup failed: {}", e.getClass().getSimpleName()); + return Optional.empty(); + } + } + + /** Public reads report a generic failure instead of internal error messages. */ + private static CommonResponse publicly(Supplier> action) + { + try { + return action.get(); + } + catch (RuntimeException e) { + log.warn("Shared report read failed: {}", e.getClass().getSimpleName()); + return CommonResponse.failure("Report data is temporarily unavailable"); + } + } + + /** Dataset reports and query charts render from saved data; earlier query reports ran SQL in the browser. */ + boolean shareable(ReportEntity report) + { + if (report.getType() == ReportType.DATASET) { + return true; + } + try { + JsonNode configure = report.getConfigure() == null ? null : mapper.readTree(report.getConfigure()); + return configure != null && configure.path("queryChart").isObject(); + } + catch (JsonProcessingException e) { + return false; + } + } + + /** One origin per line, lower case and de-duplicated; empty when the input is invalid. */ + static Optional normalizeOrigins(String value) + { + Set origins = new LinkedHashSet<>(); + for (String item : StringUtils.defaultString(value).split("[\\s,]+")) { + String origin = StringUtils.removeEnd(item.trim().toLowerCase(Locale.ROOT), "/"); + if (origin.isEmpty()) { + continue; + } + if (!ORIGIN.matcher(origin).matches()) { + return Optional.empty(); + } + origins.add(origin); + } + return origins.size() > MAX_ORIGINS ? Optional.empty() : Optional.of(String.join("\n", origins)); + } + + private ShareState state(ReportEntity report) + { + return new ShareState(report.isShareEnabled(), report.isShareEmbed(), StringUtils.defaultString(report.getShareOrigins()), report.getShareToken()); + } + + private String newToken() + { + byte[] bytes = new byte[24]; + random.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + /** Runs the saved definition with the owner's identity, so ownership checks apply as for the owner. */ + private T asOwner(ReportEntity report, Supplier action) + { + UserEntity owner = userRepository.findById(report.getUserId()).orElse(null); + SecurityContext previous = SecurityContextHolder.getContext(); + SecurityContext context = SecurityContextHolder.createEmptyContext(); + if (owner != null) { + UserDetailsService principal = new UserDetailsService(owner.getId(), owner.getCode(), owner.getUsername(), "", new ArrayList<>(), null); + context.setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); + } + SecurityContextHolder.setContext(context); + try { + return action.get(); + } + finally { + SecurityContextHolder.setContext(previous); + } + } + + @SuppressWarnings("unchecked") + private CommonResponse cached(String key, Supplier> loader) + { + long now = System.currentTimeMillis(); + Cached hit = cache.get(key); + if (hit != null && now - hit.at < CACHE_MILLIS) { + return (CommonResponse) hit.response; + } + CommonResponse response = loader.get(); + if (Boolean.TRUE.equals(response.getStatus())) { + if (cache.size() >= CACHE_SIZE) { + cache.entrySet().removeIf(entry -> now - entry.getValue().at >= CACHE_MILLIS); + if (cache.size() >= CACHE_SIZE) { + cache.clear(); + } + } + cache.put(key, new Cached(response, now)); + } + return response; + } + + private void evict(String token) + { + cache.keySet().removeIf(key -> key.endsWith(":" + token)); + } + + private static final class Cached + { + private final CommonResponse response; + private final long at; + + private Cached(CommonResponse response, long at) + { + this.response = response; + this.at = at; + } + } +} diff --git a/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java new file mode 100644 index 0000000000..f14aae8c9c --- /dev/null +++ b/core/metaforge-service/src/test/java/org/devlive/metaforge/service/service/impl/ReportShareServiceImplTest.java @@ -0,0 +1,251 @@ +package org.devlive.metaforge.service.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.metaforge.common.response.CommonResponse; +import org.devlive.metaforge.service.entity.DataSetEntity; +import org.devlive.metaforge.service.entity.ReportEntity; +import org.devlive.metaforge.service.entity.UserEntity; +import org.devlive.metaforge.service.enums.ReportType; +import org.devlive.metaforge.service.repository.DataSetRepository; +import org.devlive.metaforge.service.repository.ReportRepository; +import org.devlive.metaforge.service.repository.SourceRepository; +import org.devlive.metaforge.service.repository.UserRepository; +import org.devlive.metaforge.service.repository.jdbc.LinkRepository; +import org.devlive.metaforge.service.security.UserDetailsService; +import org.devlive.metaforge.service.service.DataSetService; +import org.devlive.metaforge.service.service.ReportService; +import org.devlive.metaforge.service.service.ReportShareService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class ReportShareServiceImplTest +{ + private static final long OWNER = 7L; + private static final String CHART = "{\"queryChart\":{\"version\":1},\"columns\":[]}"; + + private final List reports = new ArrayList<>(); + private final AtomicInteger adhocCalls = new AtomicInteger(); + private final AtomicReference adhocUser = new AtomicReference<>(); + private ReportShareServiceImpl service; + + @SuppressWarnings("unchecked") + private static T fake(Class type, java.util.function.BiFunction handler) + { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return "equals".equals(method.getName()) ? proxy == args[0] : "hashCode".equals(method.getName()) ? System.identityHashCode(proxy) : type.getSimpleName(); + } + if (method.isDefault()) { + return java.lang.invoke.MethodHandles.privateLookupIn(type, java.lang.invoke.MethodHandles.lookup()) + .unreflectSpecial(method, type).bindTo(proxy).invokeWithArguments(args == null ? new Object[0] : args); + } + return handler.apply(method.getName(), args); + }); + } + + private ReportRepository reportRepository() + { + return fake(ReportRepository.class, (name, args) -> { + switch (name) { + case "findByCode": + return reports.stream().filter(r -> Objects.equals(r.getCode(), args[0])).findFirst(); + case "findByShareToken": + return reports.stream().filter(r -> Objects.equals(r.getShareToken(), args[0])).findFirst(); + case "findById": + return reports.stream().filter(r -> Objects.equals(r.getId(), args[0])).findFirst(); + case "save": + return args[0]; + default: + throw new UnsupportedOperationException(name); + } + }); + } + + @Before + public void initialize() + { + UserEntity owner = new UserEntity(); + owner.setId(OWNER); + owner.setUsername("owner"); + UserRepository users = fake(UserRepository.class, (name, args) -> Objects.equals(args[0], OWNER) ? Optional.of(owner) : Optional.empty()); + DataSetEntity dataset = new DataSetEntity(); + dataset.setId(3L); + dataset.setCode("ds"); + DataSetRepository datasets = fake(DataSetRepository.class, (name, args) -> Optional.of(dataset)); + DataSetService dataSetService = fake(DataSetService.class, (name, args) -> { + adhocCalls.incrementAndGet(); + adhocUser.set(UserDetailsService.getUser().getId()); + return CommonResponse.success(Map.of("columns", List.of())); + }); + ReportService reportService = fake(ReportService.class, (name, args) -> CommonResponse.success(List.of())); + service = new ReportShareServiceImpl(reportRepository(), users, datasets, reportService, dataSetService, new ObjectMapper()); + signIn(OWNER); + } + + @After + public void close() + { + SecurityContextHolder.clearContext(); + } + + private static void signIn(Long id) + { + UserDetailsService principal = new UserDetailsService(id, "u" + id, "u" + id, "", new ArrayList<>(), null); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken(principal, "", principal.getAuthorities())); + } + + private ReportEntity report(String code, ReportType type, String configure) + { + ReportEntity report = new ReportEntity(); + report.setId((long) reports.size() + 1); + report.setCode(code); + report.setName(code); + report.setUserId(OWNER); + report.setType(type); + report.setConfigure(configure); + report.setDatasetId(3L); + report.setQuery("{\"columns\":[],\"limit\":10}"); + reports.add(report); + return report; + } + + private static ReportShareService.ShareRequest request(boolean enabled, boolean embed, String origins, boolean rotate) + { + return new ReportShareService.ShareRequest(enabled, embed, origins, rotate); + } + + @Test + public void onlyTheOwnerSharesAndTheLinkCanBeRotated() + { + report("chart", ReportType.QUERY, CHART); + signIn(99L); + assertFalse(service.share("chart", request(true, false, "", false)).getStatus()); + signIn(OWNER); + CommonResponse shared = service.share("chart", request(true, false, "", false)); + assertTrue(shared.getStatus()); + String token = shared.getData().getToken(); + assertTrue(ReportShareServiceImpl.TOKEN.matcher(token).matches()); + assertEquals(token, service.share("chart", request(true, false, "", false)).getData().getToken()); + String rotated = service.share("chart", request(true, false, "", true)).getData().getToken(); + assertNotEquals(token, rotated); + assertFalse("the old link stops working", service.view(token).getStatus()); + assertTrue(service.view(rotated).getStatus()); + } + + @Test + public void earlierQueryReportsCannotBeShared() + { + report("legacy", ReportType.QUERY, "{\"type\":\"LINE\"}"); + assertFalse(service.share("legacy", request(true, false, "", false)).getStatus()); + } + + @Test + public void thePublicViewHidesSqlSourceAndOwner() + { + ReportEntity report = report("chart", ReportType.QUERY, CHART); + report.setDescription("Weekly"); + String token = service.share("chart", request(true, false, "", false)).getData().getToken(); + ReportShareService.PublicReport view = service.view(token).getData(); + assertEquals("chart", view.getName()); + assertEquals(CHART, view.getConfigure()); + assertEquals("QUERY", view.getType()); + assertFalse(view.isEmbed()); + service.share("chart", request(false, false, "", false)); + assertFalse("disabled links are gone", service.view(token).getStatus()); + assertFalse(service.view("short").getStatus()); + assertFalse(service.view(null).getStatus()); + } + + @Test + public void framingFollowsTheOwnersChoice() + { + report("chart", ReportType.QUERY, CHART); + String token = service.share("chart", request(true, false, "", false)).getData().getToken(); + assertFalse(service.frameAncestors(token).isPresent()); + service.share("chart", request(true, true, "", false)); + assertEquals("*", service.frameAncestors(token).orElse(null)); + service.share("chart", request(true, true, "https://a.example.com/\nhttps://a.example.com, https://*.b.org:8443", false)); + assertEquals("'self' https://a.example.com https://*.b.org:8443", service.frameAncestors(token).orElse(null)); + assertFalse(service.share("chart", request(true, true, "javascript:alert(1)", false)).getStatus()); + assertFalse(service.share("chart", request(true, true, "https://a.com/path", false)).getStatus()); + } + + @Test + public void datasetDataRunsAsTheOwnerAndIsCached() + { + report("ds-report", ReportType.DATASET, "{}"); + String token = service.share("ds-report", request(true, false, "", false)).getData().getToken(); + SecurityContextHolder.clearContext(); + assertTrue(service.data(token).getStatus()); + assertEquals(Long.valueOf(OWNER), adhocUser.get()); + assertNull("the visitor stays anonymous afterwards", SecurityContextHolder.getContext().getAuthentication()); + assertTrue(service.data(token).getStatus()); + assertEquals("public views share one cached result", 1, adhocCalls.get()); + assertFalse("a dataset report has no chart refresh", service.refresh(token).getStatus()); + } + + @Test + public void savingAReportNeverChangesItsSharing() + { + SourceRepository sources = fake(SourceRepository.class, (name, args) -> Optional.empty()); + DataSetEntity dataset = new DataSetEntity(); + dataset.setCode("ds"); + DataSetRepository datasets = fake(DataSetRepository.class, (name, args) -> Optional.of(dataset)); + LinkRepository links = null; + ReportServiceImpl reportService = new ReportServiceImpl(reportRepository(), sources, datasets, links, new QueryChartRefresh(new ObjectMapper(), null)); + + ReportEntity created = new ReportEntity(); + created.setDataset(dataset); + created.setShareEnabled(true); + created.setShareToken("chosen-by-the-client-000000000000"); + ReportEntity saved = reportService.saveOrUpdate(fake(ReportRepository.class, (name, args) -> args[0]), created).getData(); + assertFalse(saved.isShareEnabled()); + assertNull(saved.getShareToken()); + + ReportEntity stored = report("ds-report", ReportType.DATASET, "{}"); + String token = service.share("ds-report", request(true, true, "https://a.com", false)).getData().getToken(); + ReportEntity update = new ReportEntity(); + update.setId(stored.getId()); + update.setDataset(dataset); + ReportEntity kept = reportService.saveOrUpdate(fake(ReportRepository.class, (name, args) -> args[0]), update).getData(); + assertNotNull(kept); + assertTrue(kept.isShareEnabled()); + assertTrue(kept.isShareEmbed()); + assertEquals(token, kept.getShareToken()); + assertEquals("https://a.com", kept.getShareOrigins()); + } + + @Test + public void internalErrorsNeverReachAnonymousVisitors() + { + ReportRepository broken = fake(ReportRepository.class, (name, args) -> { + throw new IllegalStateException("bad SQL grammar [SELECT secret FROM metaforge_report]"); + }); + ReportShareServiceImpl failing = new ReportShareServiceImpl(broken, null, null, null, null, new ObjectMapper()); + String token = "abcdefghijklmnopqrstuvwxyz012-_A"; + CommonResponse view = failing.view(token); + assertFalse(view.getStatus()); + assertFalse(String.valueOf(view.getMessage()).contains("SQL")); + assertFalse(failing.frameAncestors(token).isPresent()); + assertFalse(failing.data(token).getStatus()); + } +} From 2785e606c1b9d7d2269c4a1e48c4d01eb2c66d30 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Wed, 4 Nov 2026 10:17:57 -0500 Subject: [PATCH 140/232] feat(report): add report sharing and the public report page The report list gets a Share action: public access with a copyable link and a link reset, iframe embedding with allowed sites and ready-to-paste embed code, and a Public tag on shared reports. Share and embed addresses open a bare public page without sign-in; embedded pages are compact and ?title=0 hides the name. --- core/metaforge-web/src/i18n/locales/en.json | 27 ++- .../metaforge-web/src/i18n/locales/zh-cn.json | 27 ++- core/metaforge-web/src/main.ts | 4 + core/metaforge-web/src/model/report.ts | 4 + core/metaforge-web/src/router/index.ts | 2 + core/metaforge-web/src/router/shared.ts | 31 ++++ .../src/services/public-report.ts | 24 +++ core/metaforge-web/src/services/report.ts | 6 + .../views/components/visual/VisualView.vue | 22 ++- .../views/pages/admin/report/ReportHome.vue | 29 +++- .../views/pages/admin/report/ReportShare.vue | 154 ++++++++++++++++++ .../pages/admin/report/ReportShareUtils.ts | 28 ++++ .../src/views/public/PublicReport.vue | 75 +++++++++ .../metaforge-web/tests/report-share.test.mjs | 42 +++++ 14 files changed, 468 insertions(+), 7 deletions(-) create mode 100644 core/metaforge-web/src/router/shared.ts create mode 100644 core/metaforge-web/src/services/public-report.ts create mode 100644 core/metaforge-web/src/views/pages/admin/report/ReportShare.vue create mode 100644 core/metaforge-web/src/views/pages/admin/report/ReportShareUtils.ts create mode 100644 core/metaforge-web/src/views/public/PublicReport.vue create mode 100644 core/metaforge-web/tests/report-share.test.mjs diff --git a/core/metaforge-web/src/i18n/locales/en.json b/core/metaforge-web/src/i18n/locales/en.json index 355fb7bf54..a3bdba2c0c 100644 --- a/core/metaforge-web/src/i18n/locales/en.json +++ b/core/metaforge-web/src/i18n/locales/en.json @@ -3166,5 +3166,30 @@ "store.feature.export": "Data export", "store.feature.erDiagram": "ER diagrams", "store.feature.keyValue": "Key-value browsing", - "store.feature.objectStorage": "Object storage browsing and download" + "store.feature.objectStorage": "Object storage browsing and download", + "report.share.action": "Share", + "report.share.shared": "Public", + "report.share.title": "Share report: {name}", + "report.share.public": "Public access", + "report.share.publicHint": "Anyone with the link can view it without signing in. Data comes from the saved report; visitors never see its SQL or data source.", + "report.share.link": "Share link", + "report.share.linkAfterSave": "The link is created when you save.", + "report.share.copy": "Copy", + "report.share.rotate": "New link", + "report.share.embed": "Allow iframe embedding", + "report.share.embedHint": "Let other sites embed this report in an iframe.", + "report.share.origins": "Sites allowed to embed", + "report.share.originsPlaceholder": "For example https://example.com, one per line; leave empty to allow any site", + "report.share.originsInvalid": "Enter site addresses (scheme and host, optional port) such as https://example.com, at most 20", + "report.share.code": "Embed code", + "report.share.copyCode": "Copy embed code", + "report.share.legacy": "This report was saved by the earlier chart editor and cannot be shared. Save it again as a chart from the query page to share it.", + "report.share.saved": "Sharing settings saved", + "report.share.rotated": "A new link was created; the old one stopped working", + "report.share.failed": "Unable to save sharing settings", + "report.share.copied": "Copied", + "report.share.copyFailed": "Copy failed; copy it manually", + "report.share.publicTitle": "Public report", + "report.share.unavailable": "Report unavailable", + "report.share.unavailableHint": "The link has expired, or the owner turned off public access." } diff --git a/core/metaforge-web/src/i18n/locales/zh-cn.json b/core/metaforge-web/src/i18n/locales/zh-cn.json index 19f6636196..d8878e296d 100644 --- a/core/metaforge-web/src/i18n/locales/zh-cn.json +++ b/core/metaforge-web/src/i18n/locales/zh-cn.json @@ -3166,5 +3166,30 @@ "store.feature.export": "数据导出", "store.feature.erDiagram": "ER 图", "store.feature.keyValue": "键值浏览", - "store.feature.objectStorage": "对象存储浏览与下载" + "store.feature.objectStorage": "对象存储浏览与下载", + "report.share.action": "分享", + "report.share.shared": "已公开", + "report.share.title": "分享报表：{name}", + "report.share.public": "公开访问", + "report.share.publicHint": "拿到链接的任何人无需登录即可查看，数据按你保存的报表定义实时计算，访问者看不到 SQL 和数据源。", + "report.share.link": "分享链接", + "report.share.linkAfterSave": "保存后生成分享链接。", + "report.share.copy": "复制", + "report.share.rotate": "重置链接", + "report.share.embed": "允许 iframe 嵌入", + "report.share.embedHint": "允许其他网站通过 iframe 嵌入这个报表。", + "report.share.origins": "允许嵌入的网站", + "report.share.originsPlaceholder": "例如 https://example.com，每行一个；留空表示任何网站都可以嵌入", + "report.share.originsInvalid": "请填写网站地址（协议加域名，可带端口），例如 https://example.com，最多 20 个", + "report.share.code": "嵌入代码", + "report.share.copyCode": "复制嵌入代码", + "report.share.legacy": "这个报表由旧版图表保存，暂不支持分享。请在查询页重新保存为图表后再分享。", + "report.share.saved": "分享设置已保存", + "report.share.rotated": "已生成新链接，旧链接立即失效", + "report.share.failed": "分享设置保存失败", + "report.share.copied": "已复制", + "report.share.copyFailed": "复制失败，请手动复制", + "report.share.publicTitle": "公开报表", + "report.share.unavailable": "报表不可访问", + "report.share.unavailableHint": "链接已失效，或报表的所有者已关闭公开访问。" } diff --git a/core/metaforge-web/src/main.ts b/core/metaforge-web/src/main.ts index 7f4f30023f..fdbd362766 100644 --- a/core/metaforge-web/src/main.ts +++ b/core/metaforge-web/src/main.ts @@ -10,6 +10,10 @@ import router from '@/router' import i18n from '@/i18n/I18n' import { createIcons } from '@/fontawesome' import MetaforgeUI from 'metaforge-ui' +import { routeSharedReportPath } from '@/router/shared' + +// Shared report paths become the public report route before the router reads the address. +routeSharedReportPath(window.location, window.history) const app = createApp(App) createIcons(app) diff --git a/core/metaforge-web/src/model/report.ts b/core/metaforge-web/src/model/report.ts index 6cf378fc0d..e3bf495eb1 100644 --- a/core/metaforge-web/src/model/report.ts +++ b/core/metaforge-web/src/model/report.ts @@ -12,4 +12,8 @@ export interface ReportModel description?: string dataset?: DatasetModel source?: SourceModel + shareToken?: string + shareEnabled?: boolean + shareEmbed?: boolean + shareOrigins?: string } \ No newline at end of file diff --git a/core/metaforge-web/src/router/index.ts b/core/metaforge-web/src/router/index.ts index dbbce73bcc..5eb4073baf 100644 --- a/core/metaforge-web/src/router/index.ts +++ b/core/metaforge-web/src/router/index.ts @@ -5,6 +5,7 @@ import { createAuthRoute } from '@/router/auth' import { createDefaultRouter } from '@/router/default' import { createHttpRoute } from '@/router/http' import { createSetupRoute } from '@/router/setup' +import { createSharedRoute } from '@/router/shared' import SetupService from '@/services/setup' import { TokenUtils } from '@/utils/token' import { interimLocation, trackPendingNavigation } from '@/composables/useRouteRedirect' @@ -22,6 +23,7 @@ trackPendingNavigation(router) createHttpRoute(router) createAuthRoute(router) createSetupRoute(router) +createSharedRoute(router) createDefaultRouter(router) // 首次导航时检查是否已完成首次安装：未安装跳转 /setup，已安装访问 /setup 回登录页。 diff --git a/core/metaforge-web/src/router/shared.ts b/core/metaforge-web/src/router/shared.ts new file mode 100644 index 0000000000..f1b8f0fcd4 --- /dev/null +++ b/core/metaforge-web/src/router/shared.ts @@ -0,0 +1,31 @@ +import { Router } from 'vue-router' + +/** A shared report served on a real path: /share/report/{token} or /embed/report/{token}. */ +export function sharedReportPath(pathname: string): { token: string; embed: boolean } | null +{ + const match = /^\/(share|embed)\/report\/([A-Za-z0-9_-]{32})\/?$/.exec(pathname) + return match ? { token: match[2], embed: match[1] === 'embed' } : null +} + +/** + * The server answers those paths with the app document plus the framing policy of the report. + * The app uses hash routing, so the path is mapped to the public report route before the router + * starts; the path stays in the address bar so a reload keeps the same policy. + */ +export function routeSharedReportPath(location: Location, history: History): void +{ + const shared = sharedReportPath(location.pathname) + if (!shared || location.hash.startsWith('#/public/report/')) return + history.replaceState(history.state, '', `${ location.pathname }${ location.search }#/public/report/${ shared.token }${ shared.embed ? '?embed=1' : '' }`) +} + +/** Public report pages render without the app layout and need no sign-in. */ +export function createSharedRoute(router: Router): void +{ + router.addRoute({ + name: 'public-report', + path: '/public/report/:token', + meta: { title: 'report.share.publicTitle' }, + component: () => import('@/views/public/PublicReport.vue') + }) +} diff --git a/core/metaforge-web/src/services/public-report.ts b/core/metaforge-web/src/services/public-report.ts new file mode 100644 index 0000000000..ba62849a9b --- /dev/null +++ b/core/metaforge-web/src/services/public-report.ts @@ -0,0 +1,24 @@ +import { HttpUtils } from '@/utils/http' + +const DEFAULT_PATH = '/api/v1/public/report' + +/** Anonymous reads of a shared report by its share token. */ +class PublicReportService +{ + view(token: string) + { + return new HttpUtils().get(`${ DEFAULT_PATH }/${ encodeURIComponent(token) }`) + } + + refresh(token: string) + { + return new HttpUtils().get(`${ DEFAULT_PATH }/${ encodeURIComponent(token) }/refresh`) + } + + data(token: string) + { + return new HttpUtils().get(`${ DEFAULT_PATH }/${ encodeURIComponent(token) }/data`) + } +} + +export default new PublicReportService() diff --git a/core/metaforge-web/src/services/report.ts b/core/metaforge-web/src/services/report.ts index 567619d488..e3bec3fa0b 100644 --- a/core/metaforge-web/src/services/report.ts +++ b/core/metaforge-web/src/services/report.ts @@ -11,6 +11,12 @@ export class ReportService super(DEFAULT_PATH) } + /** Owner only: public link, iframe embedding, allowed sites and link rotation. */ + share(code: string, configure: { enabled: boolean, embed: boolean, origins: string, rotate: boolean }) + { + return new HttpUtils().put(`${DEFAULT_PATH}/${encodeURIComponent(code)}/share`, configure) + } + refreshChart(code: string) { return new HttpUtils().get(`${DEFAULT_PATH}/${encodeURIComponent(code)}/chart-refresh`) diff --git a/core/metaforge-web/src/views/components/visual/VisualView.vue b/core/metaforge-web/src/views/components/visual/VisualView.vue index 64f37d5ecc..ca15b555aa 100644 --- a/core/metaforge-web/src/views/components/visual/VisualView.vue +++ b/core/metaforge-web/src/views/components/visual/VisualView.vue @@ -46,6 +46,7 @@ import { computed, defineAsyncComponent, onBeforeUnmount, ref, shallowRef, watch import { MfMessage } from 'metaforge-ui' import { RefreshCw } from '@lucide/vue' import ReportService from '@/services/report' +import PublicReportService from '@/services/public-report' import type { QueryChartRow } from '@/model/query-chart' import { cloneDeep } from 'lodash' import { Type } from '@/views/components/visual/Type' @@ -69,6 +70,8 @@ const props = withDefaults(defineProps<{ original?: string reportCode?: string realtime?: boolean + /** Renders a shared report: data comes from the public API by token, never from visitor SQL. */ + shareToken?: string }>(), { width: '100%', height: '400px' @@ -82,7 +85,7 @@ const snapshot = computed(() => { return { options: saved.options, rows: config.columns } }) -const dynamicCode = computed(() => props.realtime && snapshot.value ? props.reportCode : undefined) +const dynamicCode = computed(() => props.realtime && snapshot.value ? (props.shareToken || props.reportCode) : undefined) const refreshInterval = computed(() => { const seconds = snapshot.value?.options.refreshSeconds return seconds === 60 || seconds === 300 || seconds === 900 ? seconds : 0 @@ -100,7 +103,7 @@ async function refreshChart() { const run = generation refreshing.value = true try { - const response = await ReportService.refreshChart(code) + const response = props.shareToken ? await PublicReportService.refresh(props.shareToken) : await ReportService.refreshChart(code) if (run !== generation) return if (!response.status || !Array.isArray(response.data)) { refreshFailed.value = true @@ -156,7 +159,20 @@ const handleInitialize = () => { localConfiguration.value = cloneDeep(props.configuration) as Configuration setTimeout(() => { loading.value = true - if (props.type === 'QUERY') { + if (props.shareToken) { + PublicReportService.data(props.shareToken) + .then((response) => { + if (response.status) { + formatRaw(response) + } + else { + hasError.value = true + message.value = response.message + } + }) + .finally(() => (loading.value = false)) + } + else if (props.type === 'QUERY') { const configure: ExecuteModel = { name: props.original as any, content: props.query as any, mode: 'REPORT' } ExecuteService.execute(configure, null) .then((response) => { diff --git a/core/metaforge-web/src/views/pages/admin/report/ReportHome.vue b/core/metaforge-web/src/views/pages/admin/report/ReportHome.vue index 5a09ee847a..ff31388463 100644 --- a/core/metaforge-web/src/views/pages/admin/report/ReportHome.vue +++ b/core/metaforge-web/src/views/pages/admin/report/ReportHome.vue @@ -14,7 +14,10 @@ @size-change="onSizeChange">