feat: command to export to different database

docs/commands/export.md

@@ -0,0 +1,14 @@
+# Export
+`/ledger export`
+Permission: `ledger.commands.export`
+
+---
+
+### `/ledger export <jdbc_url> [batch_size]`
+This command will export the current database to another database specified via a JDBC URL,
+skipping rows containing blacklisted items.
+
+This operation leaves the current database untouched but
+destroys any existing data in the relevant tables in the target database.
+
+Attempting to export to the same target database concurrently may lead to unexpected results.

src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt

@@ -1,6 +1,7 @@
 package com.github.quiltservertools.ledger.commands
 
 import com.github.quiltservertools.ledger.api.ExtensionManager
+import com.github.quiltservertools.ledger.commands.subcommands.ExportCommand
 import com.github.quiltservertools.ledger.commands.subcommands.InspectCommand
 import com.github.quiltservertools.ledger.commands.subcommands.PageCommand
 import com.github.quiltservertools.ledger.commands.subcommands.PlayerCommand
@@ -53,6 +54,8 @@ fun registerCommands(dispatcher: Dispatcher) {
 
     rootNode.addChild(PlayerCommand.build())
 
+    rootNode.addChild(ExportCommand.build())
+
     ExtensionManager.commands.forEach {
         it.registerSubcommands().forEach { command ->
             rootNode.addChild(command.build())

src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ExportCommand.kt

@@ -0,0 +1,56 @@
+package com.github.quiltservertools.ledger.commands.subcommands
+
+import com.github.quiltservertools.ledger.Ledger
+import com.github.quiltservertools.ledger.commands.BuildableCommand
+import com.github.quiltservertools.ledger.commands.CommandConsts
+import com.github.quiltservertools.ledger.database.DatabaseManager
+import com.github.quiltservertools.ledger.utility.Context
+import com.github.quiltservertools.ledger.utility.LiteralNode
+import com.github.quiltservertools.ledger.utility.TextColorPallet
+import com.mojang.brigadier.arguments.IntegerArgumentType
+import com.mojang.brigadier.arguments.StringArgumentType
+import kotlinx.coroutines.launch
+import me.lucko.fabric.api.permissions.v0.Permissions
+import net.minecraft.commands.Commands
+import net.minecraft.commands.Commands.literal
+import net.minecraft.network.chat.Component
+import org.jetbrains.exposed.v1.jdbc.Database
+
+object ExportCommand : BuildableCommand {
+    override fun build(): LiteralNode = literal("export")
+        .requires(Permissions.require("ledger.commands.export", CommandConsts.PERMISSION_LEVEL))
+        .then(
+            Commands.argument("jdbc_url", StringArgumentType.string()).executes {
+                runMigrate(
+                    it,
+                    StringArgumentType.getString(it, "jdbc_url"),
+                )
+            }.then(
+                Commands.argument("batch_size", IntegerArgumentType.integer(1)).executes {
+                    runMigrate(
+                        it,
+                        StringArgumentType.getString(it, "jdbc_url"),
+                        IntegerArgumentType.getInteger(it, "batch_size"),
+                    )
+                },
+            ),
+        )
+        .build()
+
+    private fun runMigrate(ctx: Context, url: String, batchSize: Int? = null): Int {
+        val source = ctx.source
+        source.sendSuccess(
+            { Component.translatable("text.ledger.export.starting").setStyle(TextColorPallet.secondary) },
+            true,
+        )
+        Ledger.launch {
+            val to = Database.connect(url)
+            DatabaseManager.exportTo(to, batchSize)
+            source.sendSuccess(
+                { Component.translatable("text.ledger.export.complete").setStyle(TextColorPallet.secondary) },
+                true,
+            )
+        }
+        return 1
+    }
+}

src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt

@@ -5,6 +5,7 @@ import com.github.quiltservertools.ledger.actions.ActionType
 import com.github.quiltservertools.ledger.actionutils.ActionSearchParams
 import com.github.quiltservertools.ledger.actionutils.Preview
 import com.github.quiltservertools.ledger.actionutils.SearchResults
+import com.github.quiltservertools.ledger.config.ActionsSpec
 import com.github.quiltservertools.ledger.config.DatabaseSpec
 import com.github.quiltservertools.ledger.config.SearchSpec
 import com.github.quiltservertools.ledger.config.config
@@ -26,6 +27,7 @@ import net.minecraft.resources.Identifier
 import net.minecraft.server.players.NameAndId
 import org.jetbrains.exposed.v1.core.Column
 import org.jetbrains.exposed.v1.core.Op
+import org.jetbrains.exposed.v1.core.ResultRow
 import org.jetbrains.exposed.v1.core.SortOrder
 import org.jetbrains.exposed.v1.core.SqlLogger
 import org.jetbrains.exposed.v1.core.Transaction
@@ -40,6 +42,7 @@ import org.jetbrains.exposed.v1.core.inSubQuery
 import org.jetbrains.exposed.v1.core.isNull
 import org.jetbrains.exposed.v1.core.lessEq
 import org.jetbrains.exposed.v1.core.neq
+import org.jetbrains.exposed.v1.core.notInList
 import org.jetbrains.exposed.v1.core.or
 import org.jetbrains.exposed.v1.core.statements.StatementContext
 import org.jetbrains.exposed.v1.core.statements.expandArgs
@@ -51,6 +54,7 @@ import org.jetbrains.exposed.v1.jdbc.Query
 import org.jetbrains.exposed.v1.jdbc.SchemaUtils
 import org.jetbrains.exposed.v1.jdbc.andWhere
 import org.jetbrains.exposed.v1.jdbc.batchInsert
+import org.jetbrains.exposed.v1.jdbc.deleteAll
 import org.jetbrains.exposed.v1.jdbc.deleteWhere
 import org.jetbrains.exposed.v1.jdbc.insertAndGetId
 import org.jetbrains.exposed.v1.jdbc.insertIgnore
@@ -62,6 +66,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction
 import org.jetbrains.exposed.v1.jdbc.update
 import org.sqlite.SQLiteConfig
 import org.sqlite.SQLiteDataSource
+import java.sql.SQLException
 import java.time.Instant
 import java.time.temporal.ChronoUnit
 import java.util.*
@@ -75,6 +80,7 @@ const val MIN_RETRY_DELAY = 1000L
 const val MAX_RETRY_DELAY = 300_000L
 private const val MAX_EXTRA_DATA_BYTES = 65_535
 
+@Suppress("LargeClass")
 object DatabaseManager {
 
     // These values are initialised late to allow the database to be created at server start,
@@ -125,7 +131,7 @@ object DatabaseManager {
         if (config[DatabaseSpec.updateSchema]) {
             try {
                 exec("CREATE INDEX IF NOT EXISTS actions_time ON actions(time)")
-            } catch (e: java.sql.SQLException) {
+            } catch (e: SQLException) {
                 logWarn("Could not create actions_time index (MySQL 8.0.12+ required if using MySQL): ${e.message}")
             }
         }
@@ -463,6 +469,151 @@ object DatabaseManager {
         }
     }
 
+    suspend fun exportTo(to: Database, batchSize: Int? = null) {
+        val batchSize = batchSize ?: config[DatabaseSpec.batchSize]
+
+        val insertOrder = arrayOf(
+            Tables.ActionIdentifiers,
+            Tables.ObjectIdentifiers,
+            Tables.Worlds,
+            Tables.Sources,
+            Tables.Players,
+            Tables.Actions,
+        )
+
+        newSuspendedTransaction(db = to) {
+            maxAttempts = MAX_QUERY_RETRIES
+            minRetryDelay = MIN_RETRY_DELAY
+            maxRetryDelay = MAX_RETRY_DELAY
+
+            if (Ledger.config[DatabaseSpec.logSQL]) {
+                addLogger(ledgerLogger)
+            }
+
+            SchemaUtils.create(
+                Tables.Players,
+                Tables.Actions,
+                Tables.ActionIdentifiers,
+                Tables.ObjectIdentifiers,
+                Tables.Sources,
+                Tables.Worlds,
+            )
+
+            for (table in insertOrder.indices.reversed().map { insertOrder[it] }) {
+                table.deleteAll()
+            }
+        }
+
+        lateinit var actionBlacklist: IntArray
+        lateinit var objectBlacklist: IntArray
+        lateinit var worldBlacklist: IntArray
+        lateinit var sourceBlacklist: IntArray
+        lateinit var playerBlacklist: IntArray
+
+        execute {
+            actionBlacklist = Tables.ActionIdentifiers.select(Tables.ActionIdentifiers.id).where {
+                Tables.ActionIdentifiers.actionIdentifier inList config[ActionsSpec.typeBlacklist]
+            }.map { it[Tables.ActionIdentifiers.id].value }.toIntArray()
+
+            val objectBlacklistStrs = config[ActionsSpec.objectBlacklist].map { toString() }
+            objectBlacklist = Tables.ObjectIdentifiers.select(Tables.ObjectIdentifiers.id).where {
+                Tables.ObjectIdentifiers.identifier inList objectBlacklistStrs
+            }.map { it[Tables.ObjectIdentifiers.id].value }.toIntArray()
+
+            val worldBlacklistStrs = config[ActionsSpec.worldBlacklist].map { toString() }
+            worldBlacklist = Tables.Worlds.select(Tables.Worlds.id).where {
+                Tables.Worlds.identifier inList worldBlacklistStrs
+            }.map { it[Tables.Worlds.id].value }.toIntArray()
+
+            sourceBlacklist = Tables.Sources.select(Tables.Sources.id).where {
+                Tables.Sources.name inList config[ActionsSpec.sourceBlacklist]
+            }.map { it[Tables.Sources.id].value }.toIntArray()
+
+            val playerBlacklistStrs = config[ActionsSpec.sourceBlacklist]
+                .filter { it.startsWith('@') }
+                .map { it.drop(1) }
+                .toList()
+            playerBlacklist = Tables.Players.select(Tables.Players.id).where {
+                Tables.Players.playerName inList playerBlacklistStrs
+            }.map { it[Tables.Players.id].value }.toIntArray()
+        }
+
+        val rows = ArrayList<ResultRow>(batchSize)
+
+        for (table in insertOrder) {
+            var start = 1
+
+            val query = table.selectAll().limit(batchSize)
+            when (table) {
+                is Tables.ActionIdentifiers -> {
+                    query.andWhere { table.id notInList actionBlacklist.asList() }
+                }
+
+                is Tables.ObjectIdentifiers -> {
+                    query.andWhere { table.id notInList objectBlacklist.asList() }
+                }
+
+                is Tables.Worlds -> {
+                    query.andWhere { table.id notInList worldBlacklist.asList() }
+                }
+
+                is Tables.Sources -> {
+                    query.andWhere { table.id notInList sourceBlacklist.asList() }
+                }
+
+                is Tables.Players -> {
+                    query.andWhere { table.id notInList playerBlacklist.asList() }
+                }
+
+                is Tables.Actions -> {
+                    query.andWhere {
+                        table.sourceName.notInList(sourceBlacklist.asList()) and
+                            table.objectId.notInList(objectBlacklist.asList()) and
+                            table.oldObjectId.notInList(objectBlacklist.asList()) and
+                            table.actionIdentifier.notInList(actionBlacklist.asList()) and
+                            table.world.notInList(worldBlacklist.asList()) and
+                            (table.sourcePlayer.isNull() or table.sourcePlayer.notInList(playerBlacklist.asList()))
+                    }
+                }
+            }
+
+            while (true) {
+                rows.clear()
+                execute {
+                    rows.addAll(
+                        query.copy().andWhere { table.id greaterEq start },
+                    )
+                }
+
+                if (!rows.isEmpty()) {
+                    newSuspendedTransaction(db = to) {
+                        maxAttempts = MAX_QUERY_RETRIES
+                        minRetryDelay = MIN_RETRY_DELAY
+                        maxRetryDelay = MAX_RETRY_DELAY
+
+                        if (Ledger.config[DatabaseSpec.logSQL]) {
+                            addLogger(ledgerLogger)
+                        }
+
+                        val inserted = table.batchInsert(rows, shouldReturnGeneratedValues = false) {
+                            for (field in table.fields) {
+                                this[field as Column<Any?>] = it[field]
+                            }
+                        }
+
+                        Ledger.logger.info("Inserted " + inserted.size + " rows into " + table.tableName)
+                    }
+                }
+
+                if (rows.size < batchSize) {
+                    break
+                }
+
+                start = rows.last().get(table.id).value + 1
+            }
+        }
+    }
+
     private fun Transaction.insertActions(actions: List<ActionType>) {
         val (safe, oversized) = actions.partition {
             it.extraData == null || it.extraData!!.length <= MAX_EXTRA_DATA_BYTES

src/main/resources/data/ledger/lang/en_us.json

@@ -76,5 +76,8 @@
     "text.ledger.purge.starting": "------ Starting purge ------",
     "text.ledger.purge.complete": "------ Completed purge ------",
 
+    "text.ledger.export.starting": "------ Starting export ------",
+    "text.ledger.export.complete": "------ Completed export ------",
+
     "text.ledger.player.result": "%1$s: First joined %2$s. Last joined: %3$s"
 }