From 9fdeefb0c1cea00619b7a2032b260c8a45f384e6 Mon Sep 17 00:00:00 2001 From: Qther Date: Mon, 14 Sep 2026 18:18:16 +0800 Subject: [PATCH 1/2] feat: command to export to different database --- docs/commands/export.md | 14 ++ .../ledger/commands/LedgerCommand.kt | 3 + .../commands/subcommands/ExportCommand.kt | 56 +++++++ .../ledger/database/DatabaseManager.kt | 151 +++++++++++++++++- .../resources/data/ledger/lang/en_us.json | 3 + 5 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 docs/commands/export.md create mode 100644 src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ExportCommand.kt diff --git a/docs/commands/export.md b/docs/commands/export.md new file mode 100644 index 00000000..6561565e --- /dev/null +++ b/docs/commands/export.md @@ -0,0 +1,14 @@ +# Export +`/ledger export` +Permission: `ledger.commands.export` + +--- + +### `/ledger export [batch_size]` +This command will export the current database to another database specified via a JDBC URL, +skipping rows containing blacklisted items. + +This operation leaves the current database untouched but +destroys any existing data in the relevant tables in the target database. + +Attempting to export to the same target database concurrently may lead to unexpected results. \ No newline at end of file diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt b/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt index 4abc30c9..1add696b 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt @@ -1,6 +1,7 @@ package com.github.quiltservertools.ledger.commands import com.github.quiltservertools.ledger.api.ExtensionManager +import com.github.quiltservertools.ledger.commands.subcommands.ExportCommand import com.github.quiltservertools.ledger.commands.subcommands.InspectCommand import com.github.quiltservertools.ledger.commands.subcommands.PageCommand import com.github.quiltservertools.ledger.commands.subcommands.PlayerCommand @@ -53,6 +54,8 @@ fun registerCommands(dispatcher: Dispatcher) { rootNode.addChild(PlayerCommand.build()) + rootNode.addChild(ExportCommand.build()) + ExtensionManager.commands.forEach { it.registerSubcommands().forEach { command -> rootNode.addChild(command.build()) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ExportCommand.kt b/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ExportCommand.kt new file mode 100644 index 00000000..ff399966 --- /dev/null +++ b/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ExportCommand.kt @@ -0,0 +1,56 @@ +package com.github.quiltservertools.ledger.commands.subcommands + +import com.github.quiltservertools.ledger.Ledger +import com.github.quiltservertools.ledger.commands.BuildableCommand +import com.github.quiltservertools.ledger.commands.CommandConsts +import com.github.quiltservertools.ledger.database.DatabaseManager +import com.github.quiltservertools.ledger.utility.Context +import com.github.quiltservertools.ledger.utility.LiteralNode +import com.github.quiltservertools.ledger.utility.TextColorPallet +import com.mojang.brigadier.arguments.IntegerArgumentType +import com.mojang.brigadier.arguments.StringArgumentType +import kotlinx.coroutines.launch +import me.lucko.fabric.api.permissions.v0.Permissions +import net.minecraft.commands.Commands +import net.minecraft.commands.Commands.literal +import net.minecraft.network.chat.Component +import org.jetbrains.exposed.v1.jdbc.Database + +object ExportCommand : BuildableCommand { + override fun build(): LiteralNode = literal("export") + .requires(Permissions.require("ledger.commands.export", CommandConsts.PERMISSION_LEVEL)) + .then( + Commands.argument("jdbc_url", StringArgumentType.string()).executes { + runMigrate( + it, + StringArgumentType.getString(it, "jdbc_url"), + ) + }.then( + Commands.argument("batch_size", IntegerArgumentType.integer(1)).executes { + runMigrate( + it, + StringArgumentType.getString(it, "jdbc_url"), + IntegerArgumentType.getInteger(it, "batch_size"), + ) + }, + ), + ) + .build() + + private fun runMigrate(ctx: Context, url: String, batchSize: Int? = null): Int { + val source = ctx.source + source.sendSuccess( + { Component.translatable("text.ledger.export.starting").setStyle(TextColorPallet.secondary) }, + true, + ) + Ledger.launch { + val to = Database.connect(url) + DatabaseManager.exportTo(to, batchSize) + source.sendSuccess( + { Component.translatable("text.ledger.export.complete").setStyle(TextColorPallet.secondary) }, + true, + ) + } + return 1 + } +} diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index bb6a5f58..a0fc148d 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -5,6 +5,7 @@ import com.github.quiltservertools.ledger.actions.ActionType import com.github.quiltservertools.ledger.actionutils.ActionSearchParams import com.github.quiltservertools.ledger.actionutils.Preview import com.github.quiltservertools.ledger.actionutils.SearchResults +import com.github.quiltservertools.ledger.config.ActionsSpec import com.github.quiltservertools.ledger.config.DatabaseSpec import com.github.quiltservertools.ledger.config.SearchSpec import com.github.quiltservertools.ledger.config.config @@ -26,6 +27,7 @@ import net.minecraft.resources.Identifier import net.minecraft.server.players.NameAndId import org.jetbrains.exposed.v1.core.Column import org.jetbrains.exposed.v1.core.Op +import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.SqlLogger import org.jetbrains.exposed.v1.core.Transaction @@ -40,6 +42,7 @@ import org.jetbrains.exposed.v1.core.inSubQuery import org.jetbrains.exposed.v1.core.isNull import org.jetbrains.exposed.v1.core.lessEq import org.jetbrains.exposed.v1.core.neq +import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.core.statements.StatementContext import org.jetbrains.exposed.v1.core.statements.expandArgs @@ -51,6 +54,7 @@ import org.jetbrains.exposed.v1.jdbc.Query import org.jetbrains.exposed.v1.jdbc.SchemaUtils import org.jetbrains.exposed.v1.jdbc.andWhere import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insertAndGetId import org.jetbrains.exposed.v1.jdbc.insertIgnore @@ -62,6 +66,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update import org.sqlite.SQLiteConfig import org.sqlite.SQLiteDataSource +import java.sql.SQLException import java.time.Instant import java.time.temporal.ChronoUnit import java.util.* @@ -75,6 +80,7 @@ const val MIN_RETRY_DELAY = 1000L const val MAX_RETRY_DELAY = 300_000L private const val MAX_EXTRA_DATA_BYTES = 65_535 +@Suppress("LargeClass") object DatabaseManager { // These values are initialised late to allow the database to be created at server start, @@ -125,7 +131,7 @@ object DatabaseManager { if (config[DatabaseSpec.updateSchema]) { try { exec("CREATE INDEX IF NOT EXISTS actions_time ON actions(time)") - } catch (e: java.sql.SQLException) { + } catch (e: SQLException) { logWarn("Could not create actions_time index (MySQL 8.0.12+ required if using MySQL): ${e.message}") } } @@ -463,6 +469,149 @@ object DatabaseManager { } } + suspend fun exportTo(to: Database, batchSize: Int? = null) { + val batchSize = batchSize ?: config[DatabaseSpec.batchSize] + + val insertOrder = arrayOf( + Tables.ActionIdentifiers, + Tables.ObjectIdentifiers, + Tables.Worlds, + Tables.Sources, + Tables.Players, + Tables.Actions, + ) + + newSuspendedTransaction(db = to) { + maxAttempts = MAX_QUERY_RETRIES + minRetryDelay = MIN_RETRY_DELAY + maxRetryDelay = MAX_RETRY_DELAY + + if (Ledger.config[DatabaseSpec.logSQL]) { + addLogger(ledgerLogger) + } + + SchemaUtils.create( + Tables.Players, + Tables.Actions, + Tables.ActionIdentifiers, + Tables.ObjectIdentifiers, + Tables.Sources, + Tables.Worlds, + ) + + for (table in insertOrder.indices.reversed().map { insertOrder[it] }) { + table.deleteAll() + } + } + + lateinit var actionBlacklist: IntArray + lateinit var objectBlacklist: IntArray + lateinit var worldBlacklist: IntArray + lateinit var sourceBlacklist: IntArray + lateinit var playerBlacklist: IntArray + + execute { + actionBlacklist = Tables.ActionIdentifiers.select(Tables.ActionIdentifiers.id).where { + Tables.ActionIdentifiers.actionIdentifier inList config[ActionsSpec.typeBlacklist] + }.map { it[Tables.ActionIdentifiers.id].value }.toIntArray() + + val objectBlacklistStrs = config[ActionsSpec.objectBlacklist].map { toString() } + objectBlacklist = Tables.ObjectIdentifiers.select(Tables.ObjectIdentifiers.id).where { + Tables.ObjectIdentifiers.identifier inList objectBlacklistStrs + }.map { it[Tables.ObjectIdentifiers.id].value }.toIntArray() + + val worldBlacklistStrs = config[ActionsSpec.worldBlacklist].map { toString() } + worldBlacklist = Tables.Worlds.select(Tables.Worlds.id).where { + Tables.Worlds.identifier inList worldBlacklistStrs + }.map { it[Tables.Worlds.id].value }.toIntArray() + + sourceBlacklist = Tables.Sources.select(Tables.Sources.id).where { + Tables.Sources.name inList config[ActionsSpec.sourceBlacklist] + }.map { it[Tables.Sources.id].value }.toIntArray() + + val playerBlacklistStrs = config[ActionsSpec.sourceBlacklist] + .filter { it.startsWith('@') } + .map { it.drop(1) } + .toList() + playerBlacklist = Tables.Players.select(Tables.Players.id).where { + Tables.Players.playerName inList playerBlacklistStrs + }.map { it[Tables.Players.id].value }.toIntArray() + } + + val rows = ArrayList(batchSize) + + for (table in insertOrder) { + var start = 1 + + val query = table.selectAll().limit(batchSize) + when (table) { + is Tables.ActionIdentifiers -> { + query.andWhere { table.id notInList actionBlacklist.asList() } + } + + is Tables.ObjectIdentifiers -> { + query.andWhere { table.id notInList objectBlacklist.asList() } + } + + is Tables.Worlds -> { + query.andWhere { table.id notInList worldBlacklist.asList() } + } + + is Tables.Sources -> { + query.andWhere { table.id notInList sourceBlacklist.asList() } + } + + is Tables.Players -> { + query.andWhere { table.id notInList playerBlacklist.asList() } + } + + is Tables.Actions -> { + query.andWhere { + table.sourceName.notInList(sourceBlacklist.asList()) and + table.objectId.notInList(objectBlacklist.asList()) and + table.actionIdentifier.notInList(actionBlacklist.asList()) and + table.world.notInList(worldBlacklist.asList()) + } + } + } + + while (true) { + rows.clear() + execute { + rows.addAll( + query.copy().andWhere { table.id greaterEq start }, + ) + } + + if (!rows.isEmpty()) { + newSuspendedTransaction(db = to) { + maxAttempts = MAX_QUERY_RETRIES + minRetryDelay = MIN_RETRY_DELAY + maxRetryDelay = MAX_RETRY_DELAY + + if (Ledger.config[DatabaseSpec.logSQL]) { + addLogger(ledgerLogger) + } + + val inserted = table.batchInsert(rows, shouldReturnGeneratedValues = false) { + for (field in table.fields) { + this[field as Column] = it[field] + } + } + + Ledger.logger.info("Inserted " + inserted.size + " rows into " + table.tableName) + } + } + + if (rows.size < batchSize) { + break + } + + start = rows.last().get(table.id).value + 1 + } + } + } + private fun Transaction.insertActions(actions: List) { val (safe, oversized) = actions.partition { it.extraData == null || it.extraData!!.length <= MAX_EXTRA_DATA_BYTES diff --git a/src/main/resources/data/ledger/lang/en_us.json b/src/main/resources/data/ledger/lang/en_us.json index b40f7d0c..a35ecd3d 100644 --- a/src/main/resources/data/ledger/lang/en_us.json +++ b/src/main/resources/data/ledger/lang/en_us.json @@ -76,5 +76,8 @@ "text.ledger.purge.starting": "------ Starting purge ------", "text.ledger.purge.complete": "------ Completed purge ------", + "text.ledger.export.starting": "------ Starting export ------", + "text.ledger.export.complete": "------ Completed export ------", + "text.ledger.player.result": "%1$s: First joined %2$s. Last joined: %3$s" } \ No newline at end of file From 83b9e5ce0ab96354a6ff2007cc45720582055a54 Mon Sep 17 00:00:00 2001 From: Qther Date: Mon, 14 Sep 2026 18:49:40 +0800 Subject: [PATCH 2/2] fix blacklisted players and oldObjectIds not being excluded --- .../quiltservertools/ledger/database/DatabaseManager.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index a0fc148d..c82c9206 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -569,8 +569,10 @@ object DatabaseManager { query.andWhere { table.sourceName.notInList(sourceBlacklist.asList()) and table.objectId.notInList(objectBlacklist.asList()) and + table.oldObjectId.notInList(objectBlacklist.asList()) and table.actionIdentifier.notInList(actionBlacklist.asList()) and - table.world.notInList(worldBlacklist.asList()) + table.world.notInList(worldBlacklist.asList()) and + (table.sourcePlayer.isNull() or table.sourcePlayer.notInList(playerBlacklist.asList())) } } }