From 04029f3dbaf6437b36ac71d3da789ba58f0b23a1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:35:58 +0200 Subject: [PATCH 01/68] fix: preserve downloader artifact response metadata --- openapi/paths/downloader.yaml | 10 ++++ .../routes/DownloaderGatewayArtifactProxy.kt | 60 +++++++++++++------ .../server/routes/DownloaderGatewayRoutes.kt | 33 +++------- .../services/DownloaderGatewayService.kt | 4 +- .../DownloaderGatewayArtifactProxyTest.kt | 60 +++++++++++++++++++ 5 files changed, 122 insertions(+), 45 deletions(-) diff --git a/openapi/paths/downloader.yaml b/openapi/paths/downloader.yaml index 5b061cef..3838e4dc 100644 --- a/openapi/paths/downloader.yaml +++ b/openapi/paths/downloader.yaml @@ -89,6 +89,16 @@ JobArtifact: '200': { description: Artifact bytes. } '206': { description: Partial artifact bytes. } '302': { description: Redirect to public artifact storage. } + '416': { description: Requested byte range is outside the artifact. } + '404': { $ref: ../components/common.yaml#/JsonError } + head: + tags: [downloader] + summary: Inspect a completed artifact without transferring its body + responses: + '200': { description: Artifact metadata. } + '206': { description: Partial artifact metadata. } + '302': { description: Redirect to public artifact storage. } + '416': { description: Requested byte range is outside the artifact. } '404': { $ref: ../components/common.yaml#/JsonError } JobCancel: parameters: diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt index e3718d67..ffd33a88 100644 --- a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt +++ b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt @@ -10,27 +10,38 @@ import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond import io.ktor.server.response.respondOutputStream import okhttp3.Response +import java.net.URI suspend fun forwardDownloaderArtifactRequest( call: ApplicationCall, gateway: DownloaderGatewayService, - response: DownloaderGatewayResponse, + method: String, + path: String, + query: String?, requestHeaders: Map, forceDownload: Boolean, ) { - val location = artifactHeader(response, HttpHeaders.Location) - if (location == null) { - call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) - return - } - - val upstream = runCatching { gateway.openFetchAbsolute(location, requestHeaders) } + val upstream = runCatching { gateway.openForward(method, path, query, requestHeaders, null) } .getOrElse { call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) return } + val artifact = if (shouldProxyArtifact(upstream)) { + val location = upstream.header(HttpHeaders.Location) + upstream.close() + if (location == null) { + call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) + return + } + runCatching { gateway.openFetchAbsolute(location, method, requestHeaders) } + .getOrElse { + call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) + return + } + } else { + upstream + } - val artifact = upstream val headers = artifactHeaders(artifact) headers.forEach { (name, value) -> if (shouldForwardArtifactResponseHeader(name, forceDownload)) { @@ -44,8 +55,10 @@ suspend fun forwardDownloaderArtifactRequest( try { call.respondOutputStream(contentType = contentType, status = status) { artifact.use { response -> - response.body.byteStream().use { input -> - input.copyTo(this, DEFAULT_BUFFER_SIZE) + if (method != "HEAD") { + response.body.byteStream().use { input -> + input.copyTo(this, DEFAULT_BUFFER_SIZE) + } } } } @@ -55,9 +68,6 @@ suspend fun forwardDownloaderArtifactRequest( } } -private fun artifactHeader(response: DownloaderGatewayResponse, name: String): String? = - response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second - private fun artifactHeaders(response: Response): List> = response.headers.names().flatMap { name -> response.headers(name).map { name to it } } @@ -69,11 +79,25 @@ private fun artifactResponse(response: Response, headers: List if (shouldForwardGatewayResponseHeader(name, forceDownload)) { call.response.headers.append(name, value, safeOnly = false) @@ -97,21 +96,3 @@ private fun isSseRequest(path: String, headers: Map): Boolean { val accept = headers.entries.firstOrNull { it.key.equals("Accept", ignoreCase = true) }?.value.orEmpty() return accept.contains("text/event-stream", ignoreCase = true) } - -private fun shouldProxyArtifact(path: String, response: dev.typetype.server.services.DownloaderGatewayResponse): Boolean { - if (!path.endsWith("/artifact")) return false - if (response.status != 302 && response.status != 307) return false - val location = headerValue(response, "Location") ?: return false - val markedInternal = headerValue(response, INTERNAL_ARTIFACT_PROXY_HEADER) == "1" - return markedInternal || isLegacyInternalHost(location) -} - -private fun headerValue(response: dev.typetype.server.services.DownloaderGatewayResponse, name: String): String? = - response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second - -private fun isLegacyInternalHost(location: String): Boolean { - val host = runCatching { URI(location).host }.getOrNull() ?: return false - return host.equals("garage", ignoreCase = true) -} - -private const val INTERNAL_ARTIFACT_PROXY_HEADER = "X-TypeType-Artifact-Proxy" diff --git a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt index c6a96745..56ef7de0 100644 --- a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt +++ b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt @@ -64,8 +64,8 @@ class DownloaderGatewayService( return httpClient.newCall(requestBuilder.build()).execute() } - fun openFetchAbsolute(url: String, headers: Map): Response { - val requestBuilder = Request.Builder().url(url).method("GET", null) + fun openFetchAbsolute(url: String, method: String, headers: Map): Response { + val requestBuilder = Request.Builder().url(url).method(method, null) headerValue(headers, "Range")?.takeIf { it.isNotBlank() }?.let { requestBuilder.addHeader("Range", it) } return client.newCall(requestBuilder.build()).execute() } diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt index 0f4c544d..2e19b99d 100644 --- a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt @@ -4,6 +4,7 @@ import com.sun.net.httpserver.HttpServer import dev.typetype.server.routes.downloaderGatewayRoutes import dev.typetype.server.services.DownloaderGatewayService import io.ktor.client.request.get +import io.ktor.client.request.head import io.ktor.client.request.header import io.ktor.client.statement.bodyAsText import io.ktor.http.HttpHeaders @@ -20,6 +21,39 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class DownloaderGatewayArtifactProxyTest { + @Test + fun `direct artifact head preserves metadata without a response body`() = testApplication { + val requestedMethod = AtomicReference() + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + requestedMethod.set(exchange.requestMethod) + exchange.responseHeaders.add(HttpHeaders.ContentType, "video/mp4") + exchange.responseHeaders.add(HttpHeaders.ContentLength, "4096") + exchange.responseHeaders.add(HttpHeaders.AcceptRanges, "bytes") + exchange.responseHeaders.add(HttpHeaders.ETag, "\"artifact-v1\"") + exchange.responseHeaders.add(HttpHeaders.LastModified, "Sat, 22 Aug 2026 10:00:00 GMT") + exchange.sendResponseHeaders(200, -1) + exchange.close() + } + upstream.start() + application { + routing { downloaderGatewayRoutes(DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}")) } + } + + try { + val response = client.head("/downloader/jobs/test/artifact") + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("HEAD", requestedMethod.get()) + assertEquals("4096", response.headers[HttpHeaders.ContentLength]) + assertEquals("bytes", response.headers[HttpHeaders.AcceptRanges]) + assertEquals("\"artifact-v1\"", response.headers[HttpHeaders.ETag]) + assertEquals("Sat, 22 Aug 2026 10:00:00 GMT", response.headers[HttpHeaders.LastModified]) + assertEquals("", response.bodyAsText()) + } finally { + upstream.stop(0) + } + } + @Test fun `internal artifact redirect streams range response`() = testApplication { val requestedRange = AtomicReference() @@ -91,6 +125,32 @@ class DownloaderGatewayArtifactProxyTest { } } + @Test + fun `direct artifact preserves unsatisfied range response`() = testApplication { + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + exchange.responseHeaders.add(HttpHeaders.ContentRange, "bytes */6") + exchange.responseHeaders.add(HttpHeaders.AcceptRanges, "bytes") + exchange.sendResponseHeaders(416, -1) + exchange.close() + } + upstream.start() + application { + routing { downloaderGatewayRoutes(DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}")) } + } + + try { + val response = client.get("/downloader/jobs/test/artifact") { + header(HttpHeaders.Range, "bytes=20-30") + } + assertEquals(HttpStatusCode.fromValue(416), response.status) + assertEquals("bytes */6", response.headers[HttpHeaders.ContentRange]) + assertEquals("bytes", response.headers[HttpHeaders.AcceptRanges]) + } finally { + upstream.stop(0) + } + } + private fun testDns(): Dns = Dns { hostname -> if (hostname == "typetype-garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) } From 7962c06875079c7ab210bc9c0070857d3475bd4c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:35:59 +0200 Subject: [PATCH 02/68] feat: define canonical portability records --- .../server/portability/PortabilityFormat.kt | 94 ++++++++++ .../server/portability/PortabilityRecord.kt | 160 ++++++++++++++++++ .../server/portability/PortabilityVideo.kt | 27 +++ 3 files changed, 281 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt new file mode 100644 index 00000000..b94685bb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt @@ -0,0 +1,94 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +enum class PortabilityFormat(val wireName: String) { + @SerialName("typetype") + TYPE_TYPE("typetype"), + @SerialName("pipepipe") + PIPE_PIPE("pipepipe"), + @SerialName("newpipe") + NEW_PIPE("newpipe"), + @SerialName("invidious") + INVIDIOUS("invidious"), + @SerialName("piped") + PIPED("piped"), + @SerialName("libretube") + LIBRE_TUBE("libretube"), + @SerialName("viewtube") + VIEW_TUBE("viewtube"), + @SerialName("materialious") + MATERIALIOUS("materialious"), + @SerialName("youtube-local") + YOUTUBE_LOCAL("youtube-local"), + @SerialName("flow") + FLOW("flow"), + @SerialName("skytube") + SKY_TUBE("skytube"), + @SerialName("grayjay") + GRAYJAY("grayjay"), + @SerialName("youtube-takeout") + YOUTUBE_TAKEOUT("youtube-takeout"), + @SerialName("opml") + OPML("opml"), +} + +@Serializable +enum class PortabilityCategory(val wireName: String) { + @SerialName("subscriptions") + SUBSCRIPTIONS("subscriptions"), + @SerialName("subscriptionGroups") + SUBSCRIPTION_GROUPS("subscriptionGroups"), + @SerialName("history") + HISTORY("history"), + @SerialName("playlists") + PLAYLISTS("playlists"), + @SerialName("watchLater") + WATCH_LATER("watchLater"), + @SerialName("favorites") + FAVORITES("favorites"), + @SerialName("progress") + PROGRESS("progress"), + @SerialName("searchHistory") + SEARCH_HISTORY("searchHistory"), + @SerialName("savedPlaylists") + SAVED_PLAYLISTS("savedPlaylists"), + @SerialName("settings") + SETTINGS("settings"), + @SerialName("contentFilters") + CONTENT_FILTERS("contentFilters"), +} + +@Serializable +enum class PortabilityDirection { + @SerialName("import") + IMPORT, + @SerialName("export") + EXPORT, +} + +@Serializable +data class PortabilityCapability( + val category: PortabilityCategory, + val directions: Set, + val fidelity: PortabilityFidelity, +) + +@Serializable +enum class PortabilityFidelity { + @SerialName("complete") + COMPLETE, + @SerialName("partial") + PARTIAL, +} + +@Serializable +data class PortabilityAdapterDescriptor( + val format: PortabilityFormat, + val adapterVersion: Int, + val capabilities: Set, + val defaultExtension: String, + val contentType: String, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt new file mode 100644 index 00000000..0f7b6238 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt @@ -0,0 +1,160 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject + +@Serializable +sealed interface PortabilityRecord { + val category: PortabilityCategory + fun stableKey(): String + fun parentKey(): String? = null +} + +@Serializable +@SerialName("subscription") +data class PortabilitySubscription( + val channelUrl: String, + val name: String = "", + val avatarUrl: String = "", + val subscribedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTIONS + override fun stableKey() = channelUrl.trim().lowercase() +} + +@Serializable +@SerialName("subscriptionGroup") +data class PortabilitySubscriptionGroup( + val name: String, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTION_GROUPS + override fun stableKey() = "group:${name.trim().lowercase()}" +} + +@Serializable +@SerialName("subscriptionGroupMembership") +data class PortabilitySubscriptionGroupMembership( + val groupName: String, + val channelUrl: String, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTION_GROUPS + override fun stableKey() = "member:${groupName.trim().lowercase()}:${channelUrl.trim().lowercase()}" + override fun parentKey() = groupName.trim().lowercase() +} + +@Serializable +@SerialName("history") +data class PortabilityHistory( + val video: PortabilityVideo, + val watchedAt: Long, + val positionSeconds: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.HISTORY + override fun stableKey() = "${video.url.trim().lowercase()}:$watchedAt" +} + +@Serializable +@SerialName("playlist") +data class PortabilityPlaylist( + val sourceId: String, + val name: String, + val description: String = "", + val createdAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PLAYLISTS + override fun stableKey() = "playlist:${sourceId.ifBlank { name }.trim().lowercase()}" +} + +@Serializable +@SerialName("playlistVideo") +data class PortabilityPlaylistVideo( + val playlistSourceId: String, + val position: Int, + val video: PortabilityVideo, + val addedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PLAYLISTS + override fun stableKey() = "playlist-video:${playlistSourceId.trim().lowercase()}:$position" + override fun parentKey() = playlistSourceId.trim().lowercase() +} + +@Serializable +@SerialName("watchLater") +data class PortabilityWatchLater( + val video: PortabilityVideo, + val addedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.WATCH_LATER + override fun stableKey() = video.url.trim().lowercase() +} + +@Serializable +@SerialName("favorite") +data class PortabilityFavorite( + val video: PortabilityVideo, + val favoritedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.FAVORITES + override fun stableKey() = video.url.trim().lowercase() +} + +@Serializable +@SerialName("progress") +data class PortabilityProgress( + val videoUrl: String, + val positionSeconds: Long, + val updatedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PROGRESS + override fun stableKey() = videoUrl.trim().lowercase() +} + +@Serializable +@SerialName("searchHistory") +data class PortabilitySearchHistory( + val term: String, + val searchedAt: Long, +) : PortabilityRecord { + override val category = PortabilityCategory.SEARCH_HISTORY + override fun stableKey() = "${term.trim().lowercase()}:$searchedAt" +} + +@Serializable +@SerialName("savedPlaylist") +data class PortabilitySavedPlaylist( + val sourceId: String, + val url: String, + val title: String = "", + val thumbnailUrl: String = "", + val uploaderName: String = "", + val streamCount: Long = 0L, + val playlistType: String = "", + val savedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.SAVED_PLAYLISTS + override fun stableKey() = url.trim().lowercase() +} + +@Serializable +@SerialName("settings") +data class PortabilitySettings( + val values: JsonObject, +) : PortabilityRecord { + override val category = PortabilityCategory.SETTINGS + override fun stableKey() = "settings" +} + +@Serializable +@SerialName("contentFilter") +data class PortabilityContentFilter( + val kind: String, + val value: String, + val label: String = "", + val imageUrl: String = "", + val createdAt: Long = 0L, + val metadata: JsonObject = JsonObject(emptyMap()), +) : PortabilityRecord { + override val category = PortabilityCategory.CONTENT_FILTERS + override fun stableKey() = "${kind.trim().lowercase()}:${value.trim().lowercase()}" +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt new file mode 100644 index 00000000..c71f3a18 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable + +@Serializable +data class PortabilityVideo( + val url: String, + val title: String = "", + val thumbnailUrl: String = "", + val durationSeconds: Long = 0L, + val channelName: String = "", + val channelUrl: String = "", + val channelAvatarUrl: String = "", + val viewCount: Long = 0L, + val publishedAt: Long = -1L, +) + +internal fun PortabilityVideo.normalized(): PortabilityVideo = copy( + url = url.trim(), + title = title.trim(), + thumbnailUrl = thumbnailUrl.trim(), + durationSeconds = durationSeconds.coerceAtLeast(0L), + channelName = channelName.trim(), + channelUrl = channelUrl.trim(), + channelAvatarUrl = channelAvatarUrl.trim(), + viewCount = viewCount.coerceAtLeast(0L), +) From 4ae3bae320fd68ff26b3bc2d19545a2e57e30ee7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:35:59 +0200 Subject: [PATCH 03/68] feat: define portability adapter contracts --- .../server/portability/PortabilityAdapter.kt | 73 +++++++++++++++++++ .../PortabilityArchiveInventory.kt | 62 ++++++++++++++++ .../portability/PortabilityInputFactory.kt | 20 +++++ .../server/portability/PortabilityJson.kt | 68 +++++++++++++++++ .../server/portability/PortabilityLimits.kt | 13 ++++ 5 files changed, 236 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt new file mode 100644 index 00000000..10dec38b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.nio.file.Path +import kotlinx.serialization.Serializable + +data class PortabilityInput( + val path: Path, + val filename: String, + val contentType: String?, + val size: Long, + val probe: ByteArray, + val archive: PortabilityArchiveInventory?, +) + +data class PortabilityDetection( + val format: PortabilityFormat, + val formatVersion: String?, + val confidence: Int, + val evidence: String, +) + +interface PortabilityRecordSink { + fun markCategory(category: PortabilityCategory) + fun write(record: PortabilityRecord): PortabilityWriteResult + fun issue(issue: PortabilityIssue) + fun putLookup(namespace: String, key: String, value: String) + fun lookup(namespace: String, key: String): String? +} + +interface PortabilityRecordSource { + fun categories(): Set + fun counts(): Map + fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) + fun forEachChild(category: PortabilityCategory, parentKey: String, block: (PortabilityRecord) -> Unit) { + forEach(category) { record -> + if (record.parentKey() == parentKey.trim().lowercase()) block(record) + } + } +} + +interface PortabilityAdapter { + val descriptor: PortabilityAdapterDescriptor + val autoDetect: Boolean get() = true + fun detect(input: PortabilityInput): PortabilityDetection? + fun decode(input: PortabilityInput, sink: PortabilityRecordSink) + fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = unsupportedExportCategories(categories) + fun encode(source: PortabilityRecordSource, output: OutputStream, categories: Set) + + private fun unsupportedExportCategories(categories: Set): List { + val supported = descriptor.capabilities + .filter { PortabilityDirection.EXPORT in it.directions } + .mapTo(hashSetOf()) { it.category } + return (categories - supported).map { category -> + PortabilityIssue(category, "unsupported_export_category", "${category.wireName} cannot be exported") + } + } +} + +data class PortabilityWriteResult( + val inserted: Boolean, +) + +@Serializable +data class PortabilityIssue( + val category: PortabilityCategory?, + val code: String, + val message: String, + val count: Long = 1L, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt new file mode 100644 index 00000000..85868142 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import java.nio.file.Path +import java.util.zip.ZipFile + +data class PortabilityArchiveEntry( + val name: String, + val compressedSize: Long, + val expandedSize: Long, +) + +data class PortabilityArchiveInventory( + val entries: List, + val expandedBytes: Long, +) { + val names: Set = entries.mapTo(linkedSetOf()) { it.name } +} + +object PortabilityArchiveInspector { + fun inspect(path: Path): PortabilityArchiveInventory? { + if (!hasZipSignature(path)) return null + ZipFile(path.toFile()).use { zip -> + val entries = ArrayList() + var expandedTotal = 0L + val iterator = zip.entries().asIterator() + while (iterator.hasNext()) { + val entry = iterator.next() + require(entries.size < PortabilityLimits.MAX_ARCHIVE_ENTRIES) { "Archive contains too many entries" } + validateName(entry.name) + if (entry.isDirectory) continue + val expanded = entry.size + val compressed = entry.compressedSize + require(expanded in 0..PortabilityLimits.MAX_ARCHIVE_ENTRY_BYTES) { "Archive entry is too large" } + expandedTotal = Math.addExact(expandedTotal, expanded) + require(expandedTotal <= PortabilityLimits.MAX_ARCHIVE_EXPANDED_BYTES) { "Archive expands beyond the limit" } + if (expanded > 0L && compressed == 0L) error("Archive entry has an invalid compression size") + if (expanded > 0L && compressed > 0L) { + require(expanded / compressed <= PortabilityLimits.MAX_COMPRESSION_RATIO) { + "Archive entry exceeds the compression ratio limit" + } + } + entries += PortabilityArchiveEntry(entry.name, compressed, expanded) + } + return PortabilityArchiveInventory(entries, expandedTotal) + } + } + + private fun hasZipSignature(path: Path): Boolean { + if (path.toFile().length() < 4L) return false + return path.toFile().inputStream().use { input -> + input.read() == 0x50 && input.read() == 0x4b + } + } + + private fun validateName(name: String) { + require(name.isNotBlank() && '\u0000' !in name) { "Archive entry has an invalid name" } + val normalized = Path.of(name.replace('\\', '/')).normalize() + require(!normalized.isAbsolute && normalized.none { it.toString() == ".." }) { + "Archive entry escapes its root" + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt new file mode 100644 index 00000000..4ef79671 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path + +object PortabilityInputFactory { + fun create(path: Path, filename: String, contentType: String?): PortabilityInput { + val size = Files.size(path) + require(size in 1..PortabilityLimits.MAX_UPLOAD_BYTES) { "Backup file size is outside the allowed range" } + val probe = Files.newInputStream(path).use { it.readNBytes(PortabilityLimits.PROBE_BYTES) } + return PortabilityInput( + path = path, + filename = filename, + contentType = contentType, + size = size, + probe = probe, + archive = PortabilityArchiveInspector.inspect(path), + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt new file mode 100644 index 00000000..8e0f00b1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import com.fasterxml.jackson.core.StreamReadConstraints +import com.fasterxml.jackson.core.StreamReadFeature +import dev.typetype.server.cache.CacheJson +import kotlinx.serialization.json.JsonElement +import java.nio.file.Files +import java.io.ByteArrayOutputStream + +internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() + .streamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(100) + .maxStringLength(PortabilityLimits.MAX_RECORD_JSON_BYTES) + .maxNumberLength(1_000) + .build(), + ) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build() + +internal inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = + Files.newInputStream(path).buffered().use { input -> + PortabilityJsonFactory.createParser(input).use(block) + } + +internal fun JsonParser.requireObject() { + require(nextToken() == JsonToken.START_OBJECT) { "Backup root must be a JSON object" } +} + +internal fun JsonParser.textOrEmpty(): String = + if (currentToken().isScalarValue) valueAsString.orEmpty() else "" + +internal fun JsonParser.longOrZero(): Long = + if (currentToken().isNumeric) longValue else valueAsString?.toLongOrNull() ?: 0L + +internal fun JsonParser.readJsonElement(): JsonElement { + val bytes = ByteArrayOutputStream() + PortabilityJsonFactory.createGenerator(bytes).use { generator -> + generator.copyCurrentStructure(this) + } + require(bytes.size() <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "JSON value is too large" } + return CacheJson.parseToJsonElement(bytes.toString(Charsets.UTF_8)) +} + +internal fun youtubeChannelUrl(value: String): String { + val trimmed = value.trim() + return when { + trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed + trimmed.isNotBlank() -> "https://www.youtube.com/channel/$trimmed" + else -> "" + } +} + +internal fun youtubeVideoUrl(value: String): String { + val trimmed = value.trim() + return when { + trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed + trimmed.isNotBlank() -> "https://www.youtube.com/watch?v=$trimmed" + else -> "" + } +} + +internal fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { + it != '&' && it != '?' && it != '#' +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt new file mode 100644 index 00000000..0d5a5aa3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt @@ -0,0 +1,13 @@ +package dev.typetype.server.portability + +object PortabilityLimits { + const val MAX_UPLOAD_BYTES = 512L * 1024 * 1024 + const val MAX_ARCHIVE_ENTRIES = 10_000 + const val MAX_ARCHIVE_EXPANDED_BYTES = 2L * 1024 * 1024 * 1024 + const val MAX_ARCHIVE_ENTRY_BYTES = 512L * 1024 * 1024 + const val MAX_COMPRESSION_RATIO = 200L + const val PROBE_BYTES = 64 * 1024 + const val MAX_RECORDS = 2_000_000L + const val MAX_RECORD_JSON_BYTES = 2 * 1024 * 1024 + const val MAX_CONTAINER_RECORDS = 100_000 +} From df8bfd5ae58fab2d037555ddcec5a39bfd39f3c3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:00 +0200 Subject: [PATCH 04/68] feat: add bounded portability spool schema --- build.gradle.kts | 1 + .../server/portability/PortabilitySpool.kt | 213 ++++++++++++++++++ .../portability/PortabilitySpoolKeys.kt | 12 + .../portability/PortabilitySpoolSchema.kt | 50 ++++ 4 files changed, 276 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt diff --git a/build.gradle.kts b/build.gradle.kts index 406be47e..eeace646 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,6 +26,7 @@ repositories { dependencies { implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) + implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.16.Final")) constraints { implementation("org.jsoup:jsoup:1.23.1") { diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt new file mode 100644 index 00000000..b64dacc6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt @@ -0,0 +1,213 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import kotlinx.serialization.encodeToString +import java.io.Closeable +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection + +class PortabilitySpool private constructor( + val path: Path, + private val connection: Connection, +) : PortabilityRecordSink, PortabilityRecordSource, Closeable { + private val insertRecord = connection.prepareStatement( + "INSERT OR IGNORE INTO records(category, stable_key, parent_key, payload) VALUES (?, ?, ?, ?)", + ) + private val markCategory = connection.prepareStatement( + "INSERT OR IGNORE INTO categories(category) VALUES (?)", + ) + private val upsertIssue = connection.prepareStatement( + """ + INSERT INTO issues(category, code, message, item_count) VALUES (?, ?, ?, ?) + ON CONFLICT(category, code, message) DO UPDATE SET item_count = item_count + excluded.item_count + """.trimIndent(), + ) + private val putLookup = connection.prepareStatement( + "INSERT OR REPLACE INTO lookups(namespace, lookup_key, value) VALUES (?, ?, ?)", + ) + private val getLookup = connection.prepareStatement( + "SELECT value FROM lookups WHERE namespace = ? AND lookup_key = ?", + ) + private var pendingWrites = 0 + private var attemptedRecords = 0L + private var closed = false + + override fun markCategory(category: PortabilityCategory) { + checkOpen() + markCategory.setString(1, category.wireName) + markCategory.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun write(record: PortabilityRecord): PortabilityWriteResult { + checkOpen() + markCategory(record.category) + require(attemptedRecords < PortabilityLimits.MAX_RECORDS) { "Backup contains too many records" } + val payload = CacheJson.encodeToString(record) + require(payload.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { + "Backup record is too large" + } + insertRecord.setString(1, record.category.wireName) + insertRecord.setString(2, portabilityStableHash(record.stableKey())) + insertRecord.setString(3, record.parentKey()?.let(::portabilityStableHash)) + insertRecord.setString(4, payload) + attemptedRecords += 1 + val inserted = insertRecord.executeUpdate() == 1 + pendingWrites += 1 + flushWhenNeeded() + return PortabilityWriteResult(inserted) + } + + override fun issue(issue: PortabilityIssue) { + checkOpen() + require(issue.code.isNotBlank() && issue.message.isNotBlank() && issue.count > 0L) + upsertIssue.setString(1, issue.category?.wireName ?: "") + upsertIssue.setString(2, issue.code) + upsertIssue.setString(3, issue.message) + upsertIssue.setLong(4, issue.count) + upsertIssue.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun putLookup(namespace: String, key: String, value: String) { + checkOpen() + require(namespace.isNotBlank() && key.isNotBlank()) + require(value.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "Lookup value is too large" } + putLookup.setString(1, namespace) + putLookup.setString(2, portabilityStableHash(key.trim().lowercase())) + putLookup.setString(3, value) + putLookup.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun lookup(namespace: String, key: String): String? { + checkOpen() + flush() + getLookup.setString(1, namespace) + getLookup.setString(2, portabilityStableHash(key.trim().lowercase())) + return getLookup.executeQuery().use { rows -> if (rows.next()) rows.getString(1) else null } + } + + override fun categories(): Set = counts().keys + + override fun counts(): Map { + flush() + connection.createStatement().use { statement -> + statement.executeQuery( + """ + SELECT categories.category, COUNT(records.ordinal) + FROM categories + LEFT JOIN records ON records.category = categories.category + GROUP BY categories.category + ORDER BY categories.category + """.trimIndent(), + ).use { rows -> + return buildMap { + while (rows.next()) { + val category = portabilityCategoryByWireName(rows.getString(1)) + put(category, rows.getLong(2)) + } + } + } + } + } + + fun duplicateCount(): Long = attemptedRecords - counts().values.sum() + + fun issues(): List { + flush() + connection.createStatement().use { statement -> + statement.executeQuery( + "SELECT category, code, message, item_count FROM issues ORDER BY rowid", + ).use { rows -> + return buildList { + while (rows.next()) { + add( + PortabilityIssue( + category = rows.getString(1) + .takeIf(String::isNotBlank) + ?.let(::portabilityCategoryByWireName), + code = rows.getString(2), + message = rows.getString(3), + count = rows.getLong(4), + ), + ) + } + } + } + } + } + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + flush() + connection.prepareStatement( + "SELECT payload FROM records WHERE category = ? ORDER BY ordinal", + ).use { statement -> + statement.setString(1, category.wireName) + statement.executeQuery().use { rows -> + while (rows.next()) block(CacheJson.decodeFromString(rows.getString(1))) + } + } + } + + override fun forEachChild( + category: PortabilityCategory, + parentKey: String, + block: (PortabilityRecord) -> Unit, + ) { + flush() + connection.prepareStatement( + "SELECT payload FROM records WHERE category = ? AND parent_key = ? ORDER BY ordinal", + ).use { statement -> + statement.setString(1, category.wireName) + statement.setString(2, portabilityStableHash(parentKey.trim().lowercase())) + statement.executeQuery().use { rows -> + while (rows.next()) block(CacheJson.decodeFromString(rows.getString(1))) + } + } + } + + fun flush() { + checkOpen() + if (pendingWrites == 0) return + connection.commit() + pendingWrites = 0 + } + + override fun close() { + if (closed) return + runCatching { flush() } + insertRecord.close() + markCategory.close() + upsertIssue.close() + putLookup.close() + getLookup.close() + connection.close() + closed = true + } + + fun delete() { + close() + Files.deleteIfExists(path) + Files.deleteIfExists(path.resolveSibling("${path.fileName}-wal")) + Files.deleteIfExists(path.resolveSibling("${path.fileName}-shm")) + } + + private fun flushWhenNeeded() { + if (pendingWrites >= COMMIT_INTERVAL) flush() + } + + private fun checkOpen() = check(!closed) { "Portability spool is closed" } + + companion object { + private const val COMMIT_INTERVAL = 500 + + fun create(directory: Path): PortabilitySpool = PortabilitySpoolSchema.create(directory) { path, connection -> + PortabilitySpool(path, connection) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt new file mode 100644 index 00000000..de0ff23d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.portability + +import java.security.MessageDigest +import java.util.HexFormat + +internal fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray(Charsets.UTF_8)) + .let(HexFormat.of()::formatHex) + +internal fun portabilityCategoryByWireName(value: String): PortabilityCategory = + PortabilityCategory.entries.firstOrNull { it.wireName == value } + ?: error("Unknown portability category: $value") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt new file mode 100644 index 00000000..7f81f08e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt @@ -0,0 +1,50 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.DriverManager + +internal object PortabilitySpoolSchema { + fun create(directory: Path, factory: (Path, Connection) -> T): T { + Files.createDirectories(directory) + val path = Files.createTempFile(directory, "portability-", ".sqlite") + val connection = DriverManager.getConnection("jdbc:sqlite:$path") + connection.createStatement().use { statement -> + statement.execute("PRAGMA journal_mode = WAL") + statement.execute("PRAGMA synchronous = NORMAL") + statement.execute("PRAGMA temp_store = MEMORY") + statement.execute( + """ + CREATE TABLE records( + ordinal INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + stable_key TEXT NOT NULL, + parent_key TEXT, + payload TEXT NOT NULL, + UNIQUE(category, stable_key) + ) + """.trimIndent(), + ) + statement.execute("CREATE INDEX records_parent_idx ON records(category, parent_key, ordinal)") + statement.execute("CREATE TABLE categories(category TEXT PRIMARY KEY)") + statement.execute( + "CREATE TABLE lookups(namespace TEXT NOT NULL, lookup_key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(namespace, lookup_key))", + ) + statement.execute( + """ + CREATE TABLE issues( + category TEXT NOT NULL, + code TEXT NOT NULL, + message TEXT NOT NULL, + item_count INTEGER NOT NULL, + UNIQUE(category, code, message) + ) + """.trimIndent(), + ) + } + connection.autoCommit = false + connection.commit() + return factory(path, connection) + } +} From 434499d28a3111b591ff97290e3a71d095d22347 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:00 +0200 Subject: [PATCH 05/68] test: cover portability archive and spool bounds --- .../PortabilityArchiveInspectorTest.kt | 43 ++++++++++++++ .../portability/PortabilitySpoolTest.kt | 59 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt new file mode 100644 index 00000000..f5963068 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class PortabilityArchiveInspectorTest { + @TempDir + lateinit var directory: Path + + @Test + fun `inventory reports safe archive entries`() { + val archive = zip("exportInfo" to "{}", "stores/subscriptions" to "[]") + val inventory = requireNotNull(PortabilityArchiveInspector.inspect(archive)) + assertEquals(setOf("exportInfo", "stores/subscriptions"), inventory.names) + assertEquals(4L, inventory.expandedBytes) + } + + @Test + fun `inventory rejects traversal entry names`() { + val archive = zip("../outside" to "invalid") + assertThrows(IllegalArgumentException::class.java) { + PortabilityArchiveInspector.inspect(archive) + } + } + + private fun zip(vararg entries: Pair): Path { + val path = Files.createTempFile(directory, "archive-", ".zip") + ZipOutputStream(Files.newOutputStream(path)).use { output -> + entries.forEach { (name, value) -> + output.putNextEntry(ZipEntry(name)) + output.write(value.toByteArray()) + output.closeEntry() + } + } + return path + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt new file mode 100644 index 00000000..807ce70a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt @@ -0,0 +1,59 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilitySpoolTest { + @TempDir + lateinit var directory: Path + + @Test + fun `spool deduplicates stable records and preserves insertion order`() { + val spool = PortabilitySpool.create(directory) + val first = PortabilitySubscription("https://youtube.com/channel/one", "One") + val duplicate = first.copy(name = "Renamed") + val second = PortabilitySubscription("https://youtube.com/channel/two", "Two") + + assertTrue(spool.write(first).inserted) + assertFalse(spool.write(duplicate).inserted) + assertTrue(spool.write(second).inserted) + + val records = mutableListOf() + spool.forEach(PortabilityCategory.SUBSCRIPTIONS, records::add) + assertEquals(listOf(first, second), records) + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 2L), spool.counts()) + assertEquals(1L, spool.duplicateCount()) + spool.delete() + } + + @Test + fun `spool aggregates matching issues`() { + val spool = PortabilitySpool.create(directory) + val issue = PortabilityIssue( + PortabilityCategory.SETTINGS, + "unsupported_setting", + "A source setting cannot be represented", + count = 2, + ) + spool.issue(issue) + spool.issue(issue.copy(count = 3)) + + assertEquals(listOf(issue.copy(count = 5)), spool.issues()) + spool.delete() + } + + @Test + fun `spool preserves present empty categories`() { + val spool = PortabilitySpool.create(directory) + + spool.markCategory(PortabilityCategory.HISTORY) + + assertEquals(mapOf(PortabilityCategory.HISTORY to 0L), spool.counts()) + assertEquals(setOf(PortabilityCategory.HISTORY), spool.categories()) + spool.delete() + } +} From 9edd22611225a085ae734e1f305e97f1ee148c70 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:00 +0200 Subject: [PATCH 06/68] feat: add owner-scoped portability jobs --- .../server/portability/PortabilityJob.kt | 98 ++++++++++++++ .../portability/PortabilityJobModels.kt | 121 ++++++++++++++++++ .../server/portability/PortabilityJobStore.kt | 55 ++++++++ 3 files changed, 274 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt new file mode 100644 index 00000000..53148d65 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -0,0 +1,98 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.Job +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicReference + +internal class PortabilityJob( + val id: String, + val ownerId: String, + val kind: PortabilityJobKind, + val directory: Path, + private val clock: () -> Long, +) { + val createdAt = clock() + private val value = AtomicReference( + PortabilityJobSnapshot(id, kind, PortabilityJobState.QUEUED, createdAt, createdAt), + ) + @Volatile + var task: Job? = null + @Volatile + var spool: PortabilitySpool? = null + @Volatile + var artifact: Path? = null + + fun snapshot(): PortabilityJobSnapshot = value.get() + + fun report(): PortabilityJobReport = value.get().let { + PortabilityJobReport(it.id, it.state, it.preview, it.result, it.errorCode) + } + + fun isTerminal(): Boolean = value.get().state in TERMINAL_STATES + + fun isCancelled(): Boolean = value.get().state == PortabilityJobState.CANCELLED + + fun isTaskComplete(): Boolean = task?.isCompleted != false + + fun updateProgress(progress: PortabilityJobProgress) { + while (true) { + val current = value.get() + if (current.state in TERMINAL_STATES) return + val next = current.copy(updatedAt = clock(), progress = progress) + if (value.compareAndSet(current, next)) return + } + } + + fun transition( + expected: Set, + state: PortabilityJobState, + preview: PortabilityPreview? = value.get().preview, + result: Map? = value.get().result, + errorCode: String? = null, + ) { + while (true) { + val current = value.get() + check(current.state in expected) { "Invalid portability transition ${current.state} -> $state" } + val next = current.copy( + state = state, + updatedAt = clock(), + preview = preview, + result = result, + progress = current.progress, + errorCode = errorCode, + ) + if (value.compareAndSet(current, next)) return + } + } + + fun tryTransition( + expected: Set, + state: PortabilityJobState, + errorCode: String? = null, + ): Boolean { + while (true) { + val current = value.get() + if (current.state !in expected) return false + val next = current.copy(state = state, updatedAt = clock(), errorCode = errorCode) + if (value.compareAndSet(current, next)) return true + } + } + + fun delete() { + task?.cancel() + spool?.delete() + if (!Files.exists(directory)) return + Files.walk(directory).use { paths -> + paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + + private companion object { + val TERMINAL_STATES = setOf( + PortabilityJobState.COMPLETED, + PortabilityJobState.FAILED, + PortabilityJobState.CANCELLED, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt new file mode 100644 index 00000000..d13b3ac5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt @@ -0,0 +1,121 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +enum class PortabilityJobKind { + @SerialName("import") + IMPORT, + @SerialName("export") + EXPORT, +} + +@Serializable +enum class PortabilityJobState { + @SerialName("queued") + QUEUED, + @SerialName("analyzing") + ANALYZING, + @SerialName("ready") + READY, + @SerialName("applying") + APPLYING, + @SerialName("encoding") + ENCODING, + @SerialName("completed") + COMPLETED, + @SerialName("failed") + FAILED, + @SerialName("cancelled") + CANCELLED, +} + +@Serializable +enum class PortabilityProgressPhase { + @SerialName("analyzing") + ANALYZING, + @SerialName("collecting") + COLLECTING, + @SerialName("applying") + APPLYING, + @SerialName("encoding") + ENCODING, +} + +@Serializable +enum class PortabilityProgressUnit { + @SerialName("records") + RECORDS, + @SerialName("categories") + CATEGORIES, + @SerialName("bytes") + BYTES, +} + +@Serializable +data class PortabilityJobProgress( + val phase: PortabilityProgressPhase, + val unit: PortabilityProgressUnit, + val processed: Long, + val total: Long? = null, +) + +@Serializable +data class PortabilityPreview( + val detection: PortabilityDetectionItem, + val counts: Map, + val duplicates: Long, + val issues: List, +) + +@Serializable +data class PortabilityDetectionItem( + val format: PortabilityFormat, + val formatVersion: String?, + val adapterVersion: Int, + val confidence: Int, + val evidence: String, +) + +@Serializable +data class PortabilityJobSnapshot( + val id: String, + val kind: PortabilityJobKind, + val state: PortabilityJobState, + val createdAt: Long, + val updatedAt: Long, + val preview: PortabilityPreview? = null, + val result: Map? = null, + val progress: PortabilityJobProgress? = null, + val errorCode: String? = null, +) + +@Serializable +data class PortabilityJobReport( + val id: String, + val state: PortabilityJobState, + val preview: PortabilityPreview? = null, + val result: Map? = null, + val errorCode: String? = null, +) + +@Serializable +data class PortabilityImportRequest( + val categories: Set, + val duplicatePolicy: PortabilityDuplicatePolicy = PortabilityDuplicatePolicy.SKIP, +) + +@Serializable +data class PortabilityExportRequest( + val format: PortabilityFormat, + val categories: Set, +) + +@Serializable +enum class PortabilityDuplicatePolicy { + @SerialName("skip") + SKIP, + @SerialName("replace") + REPLACE, +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt new file mode 100644 index 00000000..7d1aa187 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap + +internal class PortabilityJobStore( + private val root: Path, + private val clock: () -> Long = System::currentTimeMillis, + private val retentionMs: Long = DEFAULT_RETENTION_MS, +) : AutoCloseable { + private val jobs = ConcurrentHashMap() + + init { + Files.createDirectories(root) + } + + fun create(ownerId: String, kind: PortabilityJobKind): PortabilityJob { + cleanup() + val id = UUID.randomUUID().toString() + val directory = Files.createDirectory(root.resolve(id)) + return PortabilityJob(id, ownerId, kind, directory, clock).also { jobs[id] = it } + } + + fun get(ownerId: String, id: String): PortabilityJob = jobs[id] + ?.takeIf { it.ownerId == ownerId } + ?: throw PortabilityJobNotFoundException() + + fun remove(ownerId: String, id: String) { + val job = get(ownerId, id) + if (jobs.remove(id, job)) job.delete() + } + + fun cleanup() { + val oldest = clock() - retentionMs + jobs.entries.removeIf { entry -> + val expired = entry.value.isTerminal() && entry.value.isTaskComplete() && entry.value.snapshot().updatedAt < oldest + if (expired) entry.value.delete() + expired + } + } + + override fun close() { + jobs.values.forEach(PortabilityJob::delete) + jobs.clear() + Files.deleteIfExists(root) + } + + private companion object { + const val DEFAULT_RETENTION_MS = 24L * 60L * 60L * 1_000L + } +} + +class PortabilityJobNotFoundException : NoSuchElementException("Portability job not found") From 6a26c4766bac6e1ad171c3147f94a88c996121b1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:01 +0200 Subject: [PATCH 07/68] test: cover portability job ownership --- .../portability/PortabilityJobStoreTest.kt | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt new file mode 100644 index 00000000..b845d7cc --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt @@ -0,0 +1,39 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityJobStoreTest { + @TempDir + lateinit var directory: Path + + @Test + fun `cleanup never removes active jobs`() { + var now = 1_000L + val store = PortabilityJobStore(directory.resolve("jobs"), { now }, retentionMs = 100L) + val job = store.create("owner", PortabilityJobKind.EXPORT) + + now += 1_000L + store.cleanup() + + assertEquals(job.id, store.get("owner", job.id).id) + store.close() + } + + @Test + fun `cleanup removes completed jobs after retention`() { + var now = 1_000L + val store = PortabilityJobStore(directory.resolve("jobs"), { now }, retentionMs = 100L) + val job = store.create("owner", PortabilityJobKind.EXPORT) + job.transition(setOf(PortabilityJobState.QUEUED), PortabilityJobState.COMPLETED) + + now += 1_000L + store.cleanup() + + assertThrows(PortabilityJobNotFoundException::class.java) { store.get("owner", job.id) } + store.close() + } +} From ee97591b984c722e215954f957220d1b23fa20e5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:13 +0200 Subject: [PATCH 08/68] feat: report portability job progress --- .../server/portability/PortabilityDataPort.kt | 17 ++++ .../server/portability/PortabilityProgress.kt | 92 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt new file mode 100644 index 00000000..0757eb72 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.portability + +interface PortabilityDataPort { + suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + ): Map + + suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt new file mode 100644 index 00000000..265850d2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -0,0 +1,92 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CancellationException +import java.io.FilterOutputStream +import java.io.OutputStream + +internal class PortabilityProgressReporter( + private val job: PortabilityJob, + private val phase: PortabilityProgressPhase, + private val unit: PortabilityProgressUnit, + private val total: Long? = null, + private val interval: Long = 100L, +) { + private var processed = 0L + private var published = -1L + + init { + publish(force = true) + } + + fun add(count: Long = 1L) { + ensureActive() + require(count >= 0L) + processed = Math.addExact(processed, count) + publish(force = false) + } + + fun finish() { + ensureActive() + publish(force = true) + } + + fun ensureActive() { + if (job.isCancelled()) throw CancellationException("Portability job was cancelled") + } + + private fun publish(force: Boolean) { + if (!force && processed - published < interval) return + job.updateProgress(PortabilityJobProgress(phase, unit, processed, total)) + published = processed + } +} + +internal class ProgressRecordSink( + private val delegate: PortabilityRecordSink, + private val progress: PortabilityProgressReporter, +) : PortabilityRecordSink { + override fun markCategory(category: PortabilityCategory) { + progress.ensureActive() + delegate.markCategory(category) + } + + override fun write(record: PortabilityRecord): PortabilityWriteResult { + progress.ensureActive() + return delegate.write(record).also { progress.add() } + } + + override fun issue(issue: PortabilityIssue) { + progress.ensureActive() + delegate.issue(issue) + } + + override fun putLookup(namespace: String, key: String, value: String) { + progress.ensureActive() + delegate.putLookup(namespace, key, value) + } + + override fun lookup(namespace: String, key: String): String? { + progress.ensureActive() + return delegate.lookup(namespace, key) + } + + fun count(category: PortabilityCategory): Long = + (delegate as? PortabilityRecordSource)?.counts()?.get(category) ?: 0L +} + +internal class ProgressOutputStream( + output: OutputStream, + private val progress: PortabilityProgressReporter, +) : FilterOutputStream(output) { + override fun write(value: Int) { + progress.ensureActive() + out.write(value) + progress.add() + } + + override fun write(buffer: ByteArray, offset: Int, length: Int) { + progress.ensureActive() + out.write(buffer, offset, length) + progress.add(length.toLong()) + } +} From f105ded5f41c7eb595138164ff9886c6dfa98d51 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:14 +0200 Subject: [PATCH 09/68] feat: orchestrate asynchronous portability work --- .../server/portability/PortabilityEngine.kt | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt new file mode 100644 index 00000000..7a14f1a1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -0,0 +1,207 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path + +class PortabilityEngine internal constructor( + private val registry: PortabilityRegistry, + private val dataPort: PortabilityDataPort, + private val store: PortabilityJobStore, + private val scope: CoroutineScope, +) : AutoCloseable { + fun formats(): List = registry.descriptors() + + fun startImportPreview( + userId: String, + upload: Path, + filename: String, + contentType: String?, + formatHint: PortabilityFormat? = null, + ): PortabilityJobSnapshot { + val job = store.create(userId, PortabilityJobKind.IMPORT) + val saved = job.directory.resolve("upload") + try { + Files.move(upload, saved) + } catch (error: Exception) { + store.remove(userId, job.id) + throw error + } + job.task = scope.launch { analyze(job, saved, filename, contentType, formatHint) } + return job.snapshot() + } + + fun startExport( + userId: String, + format: PortabilityFormat, + categories: Set, + ): PortabilityJobSnapshot { + require(categories.isNotEmpty()) { "At least one category is required" } + val job = store.create(userId, PortabilityJobKind.EXPORT) + job.task = scope.launch { export(job, format, categories) } + return job.snapshot() + } + + fun snapshot(userId: String, id: String): PortabilityJobSnapshot = store.get(userId, id).snapshot() + + fun report(userId: String, id: String): PortabilityJobReport = store.get(userId, id).report() + + fun artifact(userId: String, id: String): Path { + val job = store.get(userId, id) + check(job.snapshot().state == PortabilityJobState.COMPLETED) { "Export is not complete" } + return requireNotNull(job.artifact) { "Export artifact is unavailable" } + } + + fun applyImport(userId: String, id: String, request: PortabilityImportRequest): PortabilityJobSnapshot { + require(request.categories.isNotEmpty()) { "At least one category is required" } + val job = store.get(userId, id) + val preview = requireNotNull(job.snapshot().preview) { "Import preview is not ready" } + require(request.categories.all { it.wireName in preview.counts }) { "A selected category is unavailable" } + job.transition(setOf(PortabilityJobState.READY), PortabilityJobState.APPLYING) + job.task = scope.launch { apply(job, request) } + return job.snapshot() + } + + fun cancel(userId: String, id: String): PortabilityJobSnapshot { + val job = store.get(userId, id) + val state = job.snapshot().state + if (state in TERMINAL_STATES) return job.snapshot() + if (job.tryTransition(ACTIVE_STATES, PortabilityJobState.CANCELLED)) job.task?.cancel() + return job.snapshot() + } + + fun delete(userId: String, id: String) { + val job = store.get(userId, id) + check(!job.isCancelled() || job.isTaskComplete()) { "Cancelled job is still stopping" } + store.remove(userId, id) + } + + override fun close() { + scope.cancel() + store.close() + } + + private suspend fun analyze( + job: PortabilityJob, + upload: Path, + filename: String, + contentType: String?, + formatHint: PortabilityFormat?, + ) { + try { + runJob(job, PortabilityJobState.ANALYZING) { + val input = withContext(Dispatchers.IO) { PortabilityInputFactory.create(upload, filename, contentType) } + val (adapter, detection) = registry.detect(input, formatHint) + val spool = PortabilitySpool.create(job.directory) + job.spool = spool + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.ANALYZING, + PortabilityProgressUnit.RECORDS, + ) + withContext(Dispatchers.IO) { adapter.decode(input, ProgressRecordSink(spool, progress)) } + progress.finish() + val preview = PortabilityPreview( + detection = detection.toItem(adapter.descriptor.adapterVersion), + counts = spool.counts().mapKeys { it.key.wireName }, + duplicates = spool.duplicateCount(), + issues = spool.issues(), + ) + job.transition(setOf(PortabilityJobState.ANALYZING), PortabilityJobState.READY, preview) + } + } finally { + Files.deleteIfExists(upload) + } + } + + private suspend fun apply(job: PortabilityJob, request: PortabilityImportRequest) { + runJob(job, null) { + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.CATEGORIES, + request.categories.size.toLong(), + interval = 1L, + ) + val result = dataPort.import(job.ownerId, requireNotNull(job.spool), request) { _, _ -> progress.add() } + progress.finish() + job.transition(setOf(PortabilityJobState.APPLYING), PortabilityJobState.COMPLETED, result = result) + } + } + + private suspend fun export(job: PortabilityJob, format: PortabilityFormat, categories: Set) { + runJob(job, PortabilityJobState.ENCODING) { + val adapter = registry.adapter(format) + val spool = PortabilitySpool.create(job.directory) + job.spool = spool + val collecting = PortabilityProgressReporter( + job, + PortabilityProgressPhase.COLLECTING, + PortabilityProgressUnit.RECORDS, + ) + dataPort.export(job.ownerId, categories, ProgressRecordSink(spool, collecting)) + collecting.finish() + val issues = adapter.assessExport(spool, categories) + require(issues.none { it.code == "unsupported_export_category" }) { "Export category is unsupported" } + val artifact = job.directory.resolve("export.${adapter.descriptor.defaultExtension}") + val encoding = PortabilityProgressReporter( + job, + PortabilityProgressPhase.ENCODING, + PortabilityProgressUnit.BYTES, + interval = 64L * 1024L, + ) + withContext(Dispatchers.IO) { + Files.newOutputStream(artifact).use { raw -> + ProgressOutputStream(raw, encoding).buffered().use { adapter.encode(spool, it, categories) } + } + } + encoding.finish() + job.artifact = artifact + job.transition( + setOf(PortabilityJobState.ENCODING), + PortabilityJobState.COMPLETED, + preview = PortabilityPreview( + PortabilityDetectionItem(format, null, adapter.descriptor.adapterVersion, 100, "Requested export format"), + spool.counts().mapKeys { it.key.wireName }, + spool.duplicateCount(), + issues, + ), + ) + } + } + + private suspend fun runJob(job: PortabilityJob, initial: PortabilityJobState?, block: suspend () -> Unit) { + try { + initial?.let { job.transition(setOf(PortabilityJobState.QUEUED), it) } + block() + } catch (error: CancellationException) { + job.tryTransition(ACTIVE_STATES, PortabilityJobState.CANCELLED) + throw error + } catch (error: Exception) { + job.tryTransition(ACTIVE_STATES, PortabilityJobState.FAILED, errorCode = portabilityErrorCode(error)) + } + } + + private companion object { + val TERMINAL_STATES = setOf( + PortabilityJobState.COMPLETED, + PortabilityJobState.FAILED, + PortabilityJobState.CANCELLED, + ) + val ACTIVE_STATES = PortabilityJobState.entries.toSet() - TERMINAL_STATES + } +} + +private fun PortabilityDetection.toItem(adapterVersion: Int) = PortabilityDetectionItem( + format, + formatVersion, + adapterVersion, + confidence, + evidence, +) From bc4a5b1f0d4dc1aeb50c6c8ca6421033039d3b12 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:14 +0200 Subject: [PATCH 10/68] test: cover portability engine lifecycle --- .../portability/PortabilityEngineTest.kt | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt new file mode 100644 index 00000000..b7b76666 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt @@ -0,0 +1,176 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Path + +class PortabilityEngineTest { + @TempDir + lateinit var directory: Path + + @Test + fun `preview is isolated by owner and can be applied`() = runBlocking { + val dataPort = FakeDataPort() + val engine = engine(dataPort) + val upload = directory.resolve("input.json") + Files.writeString(upload, "fixture") + + val started = engine.startImportPreview("owner-a", upload, "input.json", "application/json") + val ready = awaitState(engine, "owner-a", started.id, PortabilityJobState.READY) + + assertEquals(PortabilityFormat.NEW_PIPE, ready.preview?.detection?.format) + assertEquals(1L, ready.preview?.counts?.get("subscriptions")) + assertThrows(PortabilityJobNotFoundException::class.java) { + engine.snapshot("owner-b", started.id) + } + + engine.applyImport( + "owner-a", + started.id, + PortabilityImportRequest(setOf(PortabilityCategory.SUBSCRIPTIONS)), + ) + val completed = awaitState(engine, "owner-a", started.id, PortabilityJobState.COMPLETED) + assertEquals(1L, completed.result?.get("subscriptions")) + engine.close() + } + + @Test + fun `export produces an owned artifact`() = runBlocking { + val engine = engine(FakeDataPort()) + val started = engine.startExport("owner", PortabilityFormat.NEW_PIPE, setOf(PortabilityCategory.SUBSCRIPTIONS)) + val completed = awaitState(engine, "owner", started.id, PortabilityJobState.COMPLETED) + + assertNotNull(completed.preview) + assertEquals("exported", Files.readString(engine.artifact("owner", started.id))) + engine.close() + } + + @Test + fun `cancelled analysis stops before its files can be deleted`() = runBlocking { + val engine = engine(FakeDataPort(), SlowAdapter()) + val upload = directory.resolve("slow.json") + Files.writeString(upload, "fixture") + val started = engine.startImportPreview("owner", upload, "slow.json", "application/json") + var progressed = false + repeat(100) { + if (!progressed) { + progressed = (engine.snapshot("owner", started.id).progress?.processed ?: 0L) > 0L + if (!progressed) delay(5) + } + } + assertEquals(true, progressed) + + engine.cancel("owner", started.id) + repeat(100) { + if (runCatching { engine.delete("owner", started.id) }.isSuccess) { + assertThrows(PortabilityJobNotFoundException::class.java) { engine.snapshot("owner", started.id) } + engine.close() + return@runBlocking + } + delay(5) + } + error("Cancelled portability analysis did not stop") + } + + private fun engine(dataPort: PortabilityDataPort, adapter: PortabilityAdapter = FakeAdapter()): PortabilityEngine { + val store = PortabilityJobStore(directory.resolve("jobs")) + return PortabilityEngine( + PortabilityRegistry(listOf(adapter)), + dataPort, + store, + CoroutineScope(SupervisorJob() + Dispatchers.Default), + ) + } + + private suspend fun awaitState( + engine: PortabilityEngine, + owner: String, + id: String, + expected: PortabilityJobState, + ): PortabilityJobSnapshot { + repeat(100) { + val snapshot = engine.snapshot(owner, id) + if (snapshot.state == expected) return snapshot + if (snapshot.state == PortabilityJobState.FAILED) error("Portability job failed") + delay(10) + } + error("Portability job did not reach $expected") + } +} + +private class SlowAdapter : PortabilityAdapter by FakeAdapter() { + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + repeat(10_000) { index -> + Thread.sleep(1) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC$index")) + } + } +} + +private open class FakeAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.NEW_PIPE, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.COMPLETE, + ), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput) = PortabilityDetection( + PortabilityFormat.NEW_PIPE, + "1", + 100, + "test fixture", + ) + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1")) + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + output.write("exported".toByteArray()) + } +} + +private class FakeDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ): Map = source.counts().mapKeys { it.key.wireName }.also { result -> + request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1")) + onCategoryComplete(PortabilityCategory.SUBSCRIPTIONS, 1L) + } +} From 6b7efbf1ce963084c749375dbd2f3140a6d99ee1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:15 +0200 Subject: [PATCH 11/68] feat: register portability adapters --- .../portability/PortabilityEngineFactory.kt | 34 +++++++++ .../server/portability/PortabilityRegistry.kt | 72 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt new file mode 100644 index 00000000..107da5c7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CoroutineScope +import java.nio.file.Path + +object PortabilityEngineFactory { + fun create(root: Path, scope: CoroutineScope): PortabilityEngine { + val adapters = listOf( + TypeTypePortabilityAdapter(), + PipePipePortabilityAdapter(), + NewPipePortabilityAdapter(), + InvidiousPortabilityAdapter(), + PipedPortabilityAdapter(), + LibreTubePortabilityAdapter(), + ViewTubePortabilityAdapter(), + FlowPortabilityAdapter(), + GrayjayPortabilityAdapter(), + YoutubeTakeoutPortabilityAdapter(), + OpmlPortabilityAdapter(), + MaterialiousPortabilityAdapter(), + OpmlPortabilityAdapter(PortabilityFormat.SKY_TUBE, autoDetect = false), + OpmlPortabilityAdapter(PortabilityFormat.YOUTUBE_LOCAL, autoDetect = false), + ) + require(adapters.map { it.descriptor.format }.toSet() == PortabilityFormat.entries.toSet()) { + "Every portability format must have an adapter" + } + return PortabilityEngine( + registry = PortabilityRegistry(adapters), + dataPort = TypeTypePortabilityDataPort(), + store = PortabilityJobStore(root), + scope = scope, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt new file mode 100644 index 00000000..6432c09f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt @@ -0,0 +1,72 @@ +package dev.typetype.server.portability + +class PortabilityRegistry(adapters: List) { + private val adapters = adapters.toList() + private val byFormat = adapters.associateBy { it.descriptor.format } + + init { + require(adapters.isNotEmpty()) { "At least one portability adapter is required" } + require(byFormat.size == adapters.size) { "Only one adapter per format can be registered" } + } + + fun descriptors(): List = adapters.map { it.descriptor } + + fun adapter(format: PortabilityFormat): PortabilityAdapter = + byFormat[format] ?: throw UnsupportedPortabilityFormatException(format) + + fun detect( + input: PortabilityInput, + formatHint: PortabilityFormat? = null, + ): Pair { + if (formatHint != null) { + val adapter = adapter(formatHint) + val detection = adapter.detect(input) + ?: throw PortabilityFormatMismatchException(formatHint) + return adapter to detection + } + val matches = adapters.asSequence() + .filter(PortabilityAdapter::autoDetect) + .mapNotNull { adapter -> adapter.detect(input)?.let { adapter to it } } + .sortedByDescending { it.second.confidence } + .toList() + val best = matches.firstOrNull() ?: throw UnknownPortabilityFormatException() + require(best.second.confidence in 1..100) { "Adapter returned an invalid confidence" } + val second = matches.getOrNull(1) + if (second != null && best.second.confidence - second.second.confidence < MIN_CONFIDENCE_GAP) { + throw AmbiguousPortabilityFormatException(best.second, second.second) + } + return best + } + + private companion object { + const val MIN_CONFIDENCE_GAP = 10 + } +} + +sealed class PortabilityContractException(val code: String, message: String) : IllegalArgumentException(message) + +class UnknownPortabilityFormatException : + PortabilityContractException("portability_format_unknown", "Unable to detect the backup format") + +class UnsupportedPortabilityFormatException(format: PortabilityFormat) : + PortabilityContractException("portability_format_unsupported", "No adapter is registered for ${format.wireName}") + +class AmbiguousPortabilityFormatException(first: PortabilityDetection, second: PortabilityDetection) : + PortabilityContractException( + "portability_format_ambiguous", + "Backup matches both ${first.format.wireName} and ${second.format.wireName}", + ) + +class PortabilityFormatMismatchException(format: PortabilityFormat) : + PortabilityContractException("portability_format_mismatch", "Backup does not match ${format.wireName}") + +class PortabilityUploadTooLargeException : + PortabilityContractException("portability_upload_too_large", "Backup exceeds the upload limit") + +internal fun portabilityErrorCode(error: Exception): String = when (error) { + is PortabilityContractException -> error.code + is PortabilityJobNotFoundException -> "portability_job_not_found" + is IllegalStateException -> "portability_invalid_state" + is IllegalArgumentException -> "portability_invalid_input" + else -> "portability_failed" +} From 56911501c47ee74b71dc313fa5c2f65ecc3cd184 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:39 +0200 Subject: [PATCH 12/68] feat: connect portability to TypeType storage --- .../TypeTypePortabilityCoreWriter.kt | 121 ++++++++++++++++++ .../TypeTypePortabilityDataPort.kt | 45 +++++++ .../portability/TypeTypePortabilityExport.kt | 21 +++ .../portability/TypeTypePortabilityImport.kt | 24 ++++ .../portability/TypeTypePortabilityWriter.kt | 36 ++++++ 5 files changed, 247 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt new file mode 100644 index 00000000..6d133576 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt @@ -0,0 +1,121 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityCoreWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + playlists(json, source, categories) + } + + private fun subscriptions( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories && PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + if (record !is PortabilitySubscription) return@forEach + json.writeStartObject() + json.writeStringField("channelUrl", record.channelUrl) + json.writeStringField("name", record.name) + json.writeStringField("avatarUrl", record.avatarUrl) + json.writeNumberField("subscribedAt", record.subscribedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptionGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeArrayFieldStart("channelUrls") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + if (child is PortabilitySubscriptionGroupMembership) json.writeString(child.channelUrl) + } + json.writeEndArray() + json.writeNumberField("createdAt", 0L) + json.writeNumberField("updatedAt", 0L) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("history") + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record !is PortabilityHistory) return@forEach + json.writeStartObject() + json.writeStringField("url", record.video.url) + json.writeVideoFields(record.video) + json.writeNumberField("progress", record.positionSeconds) + json.writeNumberField("watchedAt", record.watchedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun playlists( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("id", record.sourceId) + json.writeStringField("name", record.name) + json.writeStringField("description", record.description) + json.writeArrayFieldStart("videos") + var count = 0 + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + if (child !is PortabilityPlaylistVideo) return@forEachChild + json.writeStartObject() + json.writeStringField("url", child.video.url) + json.writeVideoFields(child.video) + json.writeNumberField("position", child.position) + json.writeNumberField("addedAt", child.addedAt) + json.writeEndObject() + count += 1 + } + json.writeEndArray() + json.writeNumberField("videoCount", count) + json.writeNumberField("createdAt", record.createdAt) + json.writeEndObject() + } + json.writeEndArray() + } +} + +internal fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { + writeStringField("title", video.title) + writeStringField("thumbnail", video.thumbnailUrl) + writeNumberField("duration", video.durationSeconds) + writeStringField("channelName", video.channelName) + writeStringField("channelUrl", video.channelUrl) + writeStringField("channelAvatar", video.channelAvatarUrl) + writeNumberField("viewCount", video.viewCount) + writeNumberField("publishedAt", video.publishedAt) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt new file mode 100644 index 00000000..343a103b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionFeedCacheInvalidation + +class TypeTypePortabilityDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ): Map { + val result = linkedMapOf() + request.categories.sortedBy(PortabilityCategory::wireName).forEach { category -> + val imported = DatabaseFactory.query { + TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) + } + result[category.wireName] = imported + onCategoryComplete(category, imported) + } + if (PortabilityCategory.SUBSCRIPTIONS in request.categories) { + SubscriptionFeedCacheInvalidation.invalidate(userId) + } + return result + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + categories.sortedBy(PortabilityCategory::wireName).forEach { category -> + sink.markCategory(category) + DatabaseFactory.query { TypeTypePortabilityExport.write(userId, category, sink) } + onCategoryComplete(category, sinkCount(sink, category)) + } + } +} + +private fun sinkCount(sink: PortabilityRecordSink, category: PortabilityCategory): Long = when (sink) { + is ProgressRecordSink -> sink.count(category) + is PortabilityRecordSource -> sink.counts()[category] ?: 0L + else -> 0L +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt new file mode 100644 index 00000000..7078efd8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt @@ -0,0 +1,21 @@ +package dev.typetype.server.portability + +internal object TypeTypePortabilityExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + -> TypeTypePortabilityCoreExport.write(userId, category, sink) + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, + -> TypeTypePortabilityLibraryExport.write(userId, category, sink) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsExport.write(userId, sink) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterExport.write(userId, sink) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt new file mode 100644 index 00000000..1fa94f38 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.portability + +internal object TypeTypePortabilityImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long = when (category) { + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + -> TypeTypePortabilityCoreImport.write(userId, category, source, policy) + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, + -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt new file mode 100644 index 00000000..66fa7b75 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT +import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION +import java.io.OutputStream + +internal object TypeTypePortabilityWriter { + fun write( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", TYPE_TYPE_BACKUP_FORMAT) + json.writeNumberField("version", TYPE_TYPE_BACKUP_VERSION) + json.writeNumberField("exportedAt", System.currentTimeMillis()) + writeCategories(json, categories) + TypeTypePortabilityCoreWriter.write(json, source, categories) + TypeTypePortabilityLibraryWriter.write(json, source, categories) + TypeTypePortabilitySettingsWriter.write(json, source, categories) + TypeTypePortabilityFilterWriter.write(json, source, categories) + json.writeEndObject() + } + } + + private fun writeCategories(json: JsonGenerator, categories: Set) { + val legacy = categories.mapTo(linkedSetOf()) { + if (it == PortabilityCategory.SUBSCRIPTION_GROUPS) PortabilityCategory.SUBSCRIPTIONS.wireName else it.wireName + } + json.writeArrayFieldStart("categories") + legacy.sorted().forEach(json::writeString) + json.writeEndArray() + } +} From 82c82d0cf8ef63e2f4493844a5acad8ed7920402 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:39 +0200 Subject: [PATCH 13/68] feat: stream core TypeType account exports --- .../TypeTypePortabilityCoreExport.kt | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt new file mode 100644 index 00000000..08c313c3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt @@ -0,0 +1,124 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityCoreExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, sink) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, sink) + PortabilityCategory.HISTORY -> history(userId, sink) + PortabilityCategory.PLAYLISTS -> playlists(userId, sink) + else -> error("Unsupported core portability category") + } + } + + private fun subscriptions(userId: String, sink: PortabilityRecordSink) { + SubscriptionsTable.selectAll().where { SubscriptionsTable.userId eq userId } + .orderBy(SubscriptionsTable.subscribedAt, SortOrder.ASC) + .forEach { row -> + sink.write( + PortabilitySubscription( + row[SubscriptionsTable.channelUrl], + row[SubscriptionsTable.name], + row[SubscriptionsTable.avatarUrl], + row[SubscriptionsTable.subscribedAt], + ), + ) + } + } + + private fun groups(userId: String, sink: PortabilityRecordSink) { + val groups = SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt, SortOrder.ASC) + groups.forEach { row -> + val groupId = row[SubscriptionGroupsTable.id] + val name = row[SubscriptionGroupsTable.name] + sink.write(PortabilitySubscriptionGroup(name)) + SubscriptionGroupMembershipsTable.selectAll().where { + SubscriptionGroupMembershipsTable.groupId eq groupId + }.orderBy(SubscriptionGroupMembershipsTable.addedAt, SortOrder.ASC).forEach { membership -> + sink.write( + PortabilitySubscriptionGroupMembership( + name, + membership[SubscriptionGroupMembershipsTable.channelUrl], + ), + ) + } + } + } + + private fun history(userId: String, sink: PortabilityRecordSink) { + HistoryTable.selectAll().where { HistoryTable.userId eq userId } + .orderBy(HistoryTable.watchedAt, SortOrder.ASC) + .forEach { row -> + sink.write( + PortabilityHistory( + video = PortabilityVideo( + url = row[HistoryTable.url], + title = row[HistoryTable.title], + thumbnailUrl = row[HistoryTable.thumbnail], + durationSeconds = row[HistoryTable.duration], + channelName = row[HistoryTable.channelName], + channelUrl = row[HistoryTable.channelUrl], + channelAvatarUrl = row[HistoryTable.channelAvatar], + ), + watchedAt = row[HistoryTable.watchedAt], + positionSeconds = row[HistoryTable.progress], + ), + ) + } + } + + private fun playlists(userId: String, sink: PortabilityRecordSink) { + PlaylistsTable.selectAll().where { PlaylistsTable.userId eq userId } + .orderBy(PlaylistsTable.createdAt, SortOrder.ASC) + .forEach { row -> + val id = row[PlaylistsTable.id] + sink.write( + PortabilityPlaylist( + id, + row[PlaylistsTable.name], + row[PlaylistsTable.description], + row[PlaylistsTable.createdAt], + ), + ) + playlistVideos(userId, id, sink) + } + } + + private fun playlistVideos(userId: String, playlistId: String, sink: PortabilityRecordSink) { + PlaylistVideosTable.selectAll().where { + (PlaylistVideosTable.userId eq userId) and (PlaylistVideosTable.playlistId eq playlistId) + }.orderBy(PlaylistVideosTable.position, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityPlaylistVideo( + playlistId, + row[PlaylistVideosTable.position], + PortabilityVideo( + row[PlaylistVideosTable.url], + row[PlaylistVideosTable.title], + row[PlaylistVideosTable.thumbnail], + row[PlaylistVideosTable.duration], + row[PlaylistVideosTable.channelName], + row[PlaylistVideosTable.channelUrl], + row[PlaylistVideosTable.channelAvatar], + row[PlaylistVideosTable.viewCount], + row[PlaylistVideosTable.publishedAt], + ), + row[PlaylistVideosTable.addedAt], + ), + ) + } + } +} From 010a728d6aaf70157cd1f327a61504cc0fa62cb1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:40 +0200 Subject: [PATCH 14/68] feat: import core TypeType account records --- .../TypeTypePortabilityCoreImport.kt | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt new file mode 100644 index 00000000..5f867ea3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt @@ -0,0 +1,185 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.services.ChannelUrlCanonicalizer +import dev.typetype.server.services.SubscriptionMutationLock +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.nio.charset.StandardCharsets +import java.util.Locale +import java.util.UUID + +internal object TypeTypePortabilityCoreImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long = when (category) { + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy) + PortabilityCategory.HISTORY -> history(userId, source, policy) + PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy) + else -> error("Unsupported core portability category") + } + + private fun subscriptions( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + SubscriptionMutationLock.acquire(userId) + if (policy == PortabilityDuplicatePolicy.REPLACE) { + SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + if (record !is PortabilitySubscription) return@forEach + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + count += SubscriptionsTable.insertIgnore { + it[SubscriptionsTable.userId] = userId + it[SubscriptionsTable.channelUrl] = channelUrl + it[SubscriptionsTable.name] = record.name + it[SubscriptionsTable.avatarUrl] = record.avatarUrl + it[SubscriptionsTable.subscribedAt] = record.subscribedAt + }.insertedCount + } + return count + } + + private fun groups( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + SubscriptionMutationLock.acquire(userId) + if (policy == PortabilityDuplicatePolicy.REPLACE) { + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val normalized = record.name.trim().lowercase(Locale.ROOT) + require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } + count += SubscriptionGroupsTable.insertIgnore { + it[id] = stableId(userId, "group:$normalized") + it[SubscriptionGroupsTable.userId] = userId + it[name] = record.name.trim() + it[normalizedName] = normalized + it[createdAt] = 0L + it[updatedAt] = 0L + }.insertedCount + } + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroupMembership) return@forEach + val normalized = record.groupName.trim().lowercase(Locale.ROOT) + val groupId = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalized) + }.singleOrNull()?.get(SubscriptionGroupsTable.id) ?: return@forEach + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.empty().not() + if (!subscribed) return@forEach + count += SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = 0L + }.insertedCount + } + return count + } + + private fun history( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) HistoryTable.deleteWhere { HistoryTable.userId eq userId } + var count = 0L + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record !is PortabilityHistory) return@forEach + val exists = HistoryTable.selectAll().where { + (HistoryTable.userId eq userId) and + (HistoryTable.url eq record.video.url) and + (HistoryTable.watchedAt eq record.watchedAt) + }.empty().not() + if (!exists) { + count += HistoryTable.insertIgnore { + it[id] = UUID.randomUUID().toString() + it[HistoryTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[duration] = record.video.durationSeconds + it[progress] = record.positionSeconds + it[watchedAt] = record.watchedAt + }.insertedCount + } + } + return count + } + + private fun playlists( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) { + PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } + PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record is PortabilityPlaylist) count += insertPlaylist(userId, record) + } + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record is PortabilityPlaylistVideo) count += insertPlaylistVideo(userId, record) + } + return count + } + + private fun insertPlaylist(userId: String, record: PortabilityPlaylist): Int = PlaylistsTable.insertIgnore { + it[id] = stableId(userId, "playlist:${record.sourceId}") + it[PlaylistsTable.userId] = userId + it[name] = record.name + it[description] = record.description + it[createdAt] = record.createdAt + }.insertedCount + + private fun insertPlaylistVideo(userId: String, record: PortabilityPlaylistVideo): Int = + PlaylistVideosTable.insertIgnore { + it[id] = stableId(userId, "playlist:${record.playlistSourceId}:${record.position}:${record.video.url}") + it[playlistId] = stableId(userId, "playlist:${record.playlistSourceId}") + it[PlaylistVideosTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[position] = record.position + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[addedAt] = record.addedAt + it[publishedAt] = record.video.publishedAt + }.insertedCount +} + +private fun stableId(userId: String, value: String): String = UUID.nameUUIDFromBytes( + "$userId:$value".toByteArray(StandardCharsets.UTF_8), +).toString() From a46a388bc71fb50de083b9c8d4ea8171ed45b341 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:40 +0200 Subject: [PATCH 15/68] feat: transfer TypeType content filters --- .../TypeTypePortabilityFilterExport.kt | 103 ++++++++++++++++++ .../TypeTypePortabilityFilterImport.kt | 103 ++++++++++++++++++ .../TypeTypePortabilityFilterWriter.kt | 53 +++++++++ 3 files changed, 259 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt new file mode 100644 index 00000000..a575fa7b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityFilterExport { + private const val USER_SCOPE = "user" + + fun write(userId: String, sink: PortabilityRecordSink) { + blockedChannels(userId, sink) + blockedVideos(userId, sink) + blockedKeywords(userId, sink) + allowedChannels(userId, sink) + allowedPlaylists(userId, sink) + } + + private fun blockedChannels(userId: String, sink: PortabilityRecordSink) { + BlockedChannelsTable.selectAll().where { + (BlockedChannelsTable.userId eq userId) and (BlockedChannelsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedChannel", + row[BlockedChannelsTable.channelUrl], + row[BlockedChannelsTable.channelName].orEmpty(), + row[BlockedChannelsTable.channelThumbnailUrl].orEmpty(), + row[BlockedChannelsTable.blockedAt], + ), + ) + } + } + + private fun blockedVideos(userId: String, sink: PortabilityRecordSink) { + BlockedVideosTable.selectAll().where { + (BlockedVideosTable.userId eq userId) and (BlockedVideosTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedVideo", + row[BlockedVideosTable.videoUrl], + createdAt = row[BlockedVideosTable.blockedAt], + ), + ) + } + } + + private fun blockedKeywords(userId: String, sink: PortabilityRecordSink) { + BlockedKeywordsTable.selectAll().where { + (BlockedKeywordsTable.userId eq userId) and (BlockedKeywordsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedKeyword", + row[BlockedKeywordsTable.keyword], + createdAt = row[BlockedKeywordsTable.blockedAt], + ), + ) + } + } + + private fun allowedChannels(userId: String, sink: PortabilityRecordSink) { + AllowedChannelsTable.selectAll().where { + (AllowedChannelsTable.userId eq userId) and (AllowedChannelsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "allowedChannel", + row[AllowedChannelsTable.channelUrl], + row[AllowedChannelsTable.channelName].orEmpty(), + row[AllowedChannelsTable.channelThumbnailUrl].orEmpty(), + row[AllowedChannelsTable.allowedAt], + ), + ) + } + } + + private fun allowedPlaylists(userId: String, sink: PortabilityRecordSink) { + AllowedPlaylistsTable.selectAll().where { + (AllowedPlaylistsTable.userId eq userId) and (AllowedPlaylistsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "allowedPlaylist", + row[AllowedPlaylistsTable.playlistUrl], + row[AllowedPlaylistsTable.title].orEmpty(), + row[AllowedPlaylistsTable.thumbnailUrl].orEmpty(), + row[AllowedPlaylistsTable.allowedAt], + buildJsonObject { + put("uploaderName", row[AllowedPlaylistsTable.uploaderName].orEmpty()) + }, + ), + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt new file mode 100644 index 00000000..819dcc67 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import dev.typetype.server.services.normalizeBlockedKeyword +import dev.typetype.server.services.normalizeChannelKey +import dev.typetype.server.services.normalizePlaylistKey +import kotlinx.serialization.json.jsonPrimitive +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityFilterImport { + private const val USER_SCOPE = "user" + + fun write( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) clear(userId) + var count = 0L + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + if (record is PortabilityContentFilter) count += insert(userId, record) + } + return count + } + + private fun insert(userId: String, record: PortabilityContentFilter): Int = when (record.kind) { + "blockedChannel" -> BlockedChannelsTable.insertIgnore { + it[BlockedChannelsTable.userId] = userId + it[scope] = USER_SCOPE + it[channelUrl] = record.value + it[channelName] = record.label + it[channelThumbnailUrl] = record.imageUrl + it[blockedAt] = record.createdAt + }.insertedCount + "blockedVideo" -> BlockedVideosTable.insertIgnore { + it[BlockedVideosTable.userId] = userId + it[scope] = USER_SCOPE + it[videoUrl] = record.value + it[blockedAt] = record.createdAt + }.insertedCount + "blockedKeyword" -> BlockedKeywordsTable.insertIgnore { + it[BlockedKeywordsTable.userId] = userId + it[scope] = USER_SCOPE + it[keyword] = normalizeBlockedKeyword(record.value) + it[blockedAt] = record.createdAt + }.insertedCount + "allowedChannel" -> AllowedChannelsTable.insertIgnore { + it[AllowedChannelsTable.userId] = userId + it[scope] = USER_SCOPE + it[channelUrl] = normalizeChannelKey(record.value) + it[channelName] = record.label + it[channelThumbnailUrl] = record.imageUrl + it[allowedAt] = record.createdAt + }.insertedCount + "allowedPlaylist" -> allowedPlaylist(userId, record) + else -> throw IllegalArgumentException("Unsupported content filter kind") + } + + private fun allowedPlaylist(userId: String, record: PortabilityContentFilter): Int { + val url = normalizePlaylistKey(record.value) + val exists = AllowedPlaylistsTable.selectAll().where { + (AllowedPlaylistsTable.userId eq userId) and + (AllowedPlaylistsTable.scope eq USER_SCOPE) and + (AllowedPlaylistsTable.playlistUrl eq url) + }.empty().not() + if (exists) return 0 + return AllowedPlaylistsTable.insertIgnore { + it[AllowedPlaylistsTable.userId] = userId + it[scope] = USER_SCOPE + it[playlistUrl] = url + it[title] = record.label + it[thumbnailUrl] = record.imageUrl + it[uploaderName] = record.metadata["uploaderName"]?.jsonPrimitive?.content.orEmpty() + it[allowedAt] = record.createdAt + }.insertedCount + } + + private fun clear(userId: String) { + BlockedChannelsTable.deleteWhere { + (BlockedChannelsTable.userId eq userId) and (BlockedChannelsTable.scope eq USER_SCOPE) + } + BlockedVideosTable.deleteWhere { + (BlockedVideosTable.userId eq userId) and (BlockedVideosTable.scope eq USER_SCOPE) + } + BlockedKeywordsTable.deleteWhere { + (BlockedKeywordsTable.userId eq userId) and (BlockedKeywordsTable.scope eq USER_SCOPE) + } + AllowedChannelsTable.deleteWhere { + (AllowedChannelsTable.userId eq userId) and (AllowedChannelsTable.scope eq USER_SCOPE) + } + AllowedPlaylistsTable.deleteWhere { + (AllowedPlaylistsTable.userId eq userId) and (AllowedPlaylistsTable.scope eq USER_SCOPE) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt new file mode 100644 index 00000000..5e37bb41 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityFilterWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.CONTENT_FILTERS !in categories) return + json.writeObjectFieldStart("contentFilters") + writeArray(json, source, "blockedChannels", "blockedChannel") + writeArray(json, source, "blockedVideos", "blockedVideo") + writeArray(json, source, "blockedKeywords", "blockedKeyword") + writeArray(json, source, "allowedChannels", "allowedChannel") + writeArray(json, source, "allowedPlaylists", "allowedPlaylist") + json.writeEndObject() + } + + private fun writeArray( + json: JsonGenerator, + source: PortabilityRecordSource, + field: String, + kind: String, + ) { + json.writeArrayFieldStart(field) + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + if (record !is PortabilityContentFilter || record.kind != kind) return@forEach + json.writeStartObject() + when (kind) { + "blockedKeyword" -> json.writeStringField("keyword", record.value) + else -> json.writeStringField("url", record.value) + } + if (kind == "allowedPlaylist") { + json.writeStringField("title", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeStringField("uploaderName", record.metadata["uploaderName"]?.toString()?.trim('"').orEmpty()) + json.writeNumberField("allowedAt", record.createdAt) + } else if (kind.startsWith("allowed")) { + json.writeStringField("name", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeNumberField("allowedAt", record.createdAt) + } else { + json.writeStringField("name", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeNumberField("blockedAt", record.createdAt) + } + json.writeEndObject() + } + json.writeEndArray() + } +} From ef3f091286dd215e08dc2a30c80c4e8d656f6b6c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:41 +0200 Subject: [PATCH 16/68] feat: transfer TypeType library records --- .../TypeTypePortabilityLibraryExport.kt | 110 ++++++++++++++++ .../TypeTypePortabilityLibraryImport.kt | 121 ++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt new file mode 100644 index 00000000..fb7d4dd3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.WatchLaterTable +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityLibraryExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.WATCH_LATER -> watchLater(userId, sink) + PortabilityCategory.FAVORITES -> favorites(userId, sink) + PortabilityCategory.PROGRESS -> progress(userId, sink) + PortabilityCategory.SEARCH_HISTORY -> searchHistory(userId, sink) + PortabilityCategory.SAVED_PLAYLISTS -> savedPlaylists(userId, sink) + else -> error("Unsupported library portability category") + } + } + + private fun watchLater(userId: String, sink: PortabilityRecordSink) { + WatchLaterTable.selectAll().where { WatchLaterTable.userId eq userId } + .orderBy(WatchLaterTable.addedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityWatchLater( + PortabilityVideo( + row[WatchLaterTable.url], + row[WatchLaterTable.title], + row[WatchLaterTable.thumbnail], + row[WatchLaterTable.duration], + row[WatchLaterTable.channelName], + row[WatchLaterTable.channelUrl], + row[WatchLaterTable.channelAvatar], + row[WatchLaterTable.viewCount], + row[WatchLaterTable.publishedAt], + ), + row[WatchLaterTable.addedAt], + ), + ) + } + } + + private fun favorites(userId: String, sink: PortabilityRecordSink) { + FavoritesTable.selectAll().where { FavoritesTable.userId eq userId } + .orderBy(FavoritesTable.favoritedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityFavorite( + PortabilityVideo( + row[FavoritesTable.videoUrl], + row[FavoritesTable.title], + row[FavoritesTable.thumbnail], + row[FavoritesTable.duration], + row[FavoritesTable.channelName], + row[FavoritesTable.channelUrl], + row[FavoritesTable.channelAvatar], + row[FavoritesTable.viewCount], + row[FavoritesTable.publishedAt], + ), + row[FavoritesTable.favoritedAt], + ), + ) + } + } + + private fun progress(userId: String, sink: PortabilityRecordSink) { + ProgressTable.selectAll().where { ProgressTable.userId eq userId } + .orderBy(ProgressTable.updatedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityProgress( + row[ProgressTable.videoUrl], + row[ProgressTable.position], + row[ProgressTable.updatedAt], + ), + ) + } + } + + private fun searchHistory(userId: String, sink: PortabilityRecordSink) { + SearchHistoryTable.selectAll().where { SearchHistoryTable.userId eq userId } + .orderBy(SearchHistoryTable.searchedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilitySearchHistory( + row[SearchHistoryTable.term], + row[SearchHistoryTable.searchedAt], + ), + ) + } + } + + private fun savedPlaylists(userId: String, sink: PortabilityRecordSink) { + SavedPlaylistsTable.selectAll().where { SavedPlaylistsTable.userId eq userId } + .orderBy(SavedPlaylistsTable.savedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilitySavedPlaylist( + row[SavedPlaylistsTable.publicPlaylistId], + row[SavedPlaylistsTable.url], + row[SavedPlaylistsTable.title], + row[SavedPlaylistsTable.thumbnailUrl], + row[SavedPlaylistsTable.uploaderName], + row[SavedPlaylistsTable.streamCount], + row[SavedPlaylistsTable.playlistType], + row[SavedPlaylistsTable.savedAt], + ), + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt new file mode 100644 index 00000000..d1b3ae65 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt @@ -0,0 +1,121 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.WatchLaterTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.nio.charset.StandardCharsets +import java.util.UUID + +internal object TypeTypePortabilityLibraryImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + clearIfReplacing(userId, category, policy) + var count = 0L + source.forEach(category) { record -> + count += when (record) { + is PortabilityWatchLater -> watchLater(userId, record) + is PortabilityFavorite -> favorite(userId, record) + is PortabilityProgress -> progress(userId, record) + is PortabilitySearchHistory -> searchHistory(userId, record) + is PortabilitySavedPlaylist -> savedPlaylist(userId, record) + else -> 0 + } + } + return count + } + + private fun clearIfReplacing( + userId: String, + category: PortabilityCategory, + policy: PortabilityDuplicatePolicy, + ) { + if (policy != PortabilityDuplicatePolicy.REPLACE) return + when (category) { + PortabilityCategory.WATCH_LATER -> WatchLaterTable.deleteWhere { WatchLaterTable.userId eq userId } + PortabilityCategory.FAVORITES -> FavoritesTable.deleteWhere { FavoritesTable.userId eq userId } + PortabilityCategory.PROGRESS -> ProgressTable.deleteWhere { ProgressTable.userId eq userId } + PortabilityCategory.SEARCH_HISTORY -> SearchHistoryTable.deleteWhere { SearchHistoryTable.userId eq userId } + PortabilityCategory.SAVED_PLAYLISTS -> SavedPlaylistsTable.deleteWhere { SavedPlaylistsTable.userId eq userId } + else -> error("Unsupported library portability category") + } + } + + private fun watchLater(userId: String, record: PortabilityWatchLater): Int = WatchLaterTable.insertIgnore { + it[WatchLaterTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[addedAt] = record.addedAt + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[publishedAt] = record.video.publishedAt + }.insertedCount + + private fun favorite(userId: String, record: PortabilityFavorite): Int = FavoritesTable.insertIgnore { + it[FavoritesTable.userId] = userId + it[videoUrl] = record.video.url + it[favoritedAt] = record.favoritedAt + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[publishedAt] = record.video.publishedAt + }.insertedCount + + private fun progress(userId: String, record: PortabilityProgress): Int = ProgressTable.insertIgnore { + it[ProgressTable.userId] = userId + it[videoUrl] = record.videoUrl + it[position] = record.positionSeconds.coerceAtLeast(0L) + it[updatedAt] = record.updatedAt + }.insertedCount + + private fun searchHistory(userId: String, record: PortabilitySearchHistory): Int { + val exists = SearchHistoryTable.selectAll().where { + (SearchHistoryTable.userId eq userId) and + (SearchHistoryTable.term eq record.term) and + (SearchHistoryTable.searchedAt eq record.searchedAt) + }.empty().not() + if (exists) return 0 + return SearchHistoryTable.insertIgnore { + it[id] = UUID.randomUUID().toString() + it[SearchHistoryTable.userId] = userId + it[term] = record.term + it[searchedAt] = record.searchedAt + }.insertedCount + } + + private fun savedPlaylist(userId: String, record: PortabilitySavedPlaylist): Int = + SavedPlaylistsTable.insertIgnore { + it[id] = stableLibraryId(userId, record.url) + it[SavedPlaylistsTable.userId] = userId + it[publicPlaylistId] = record.sourceId + it[url] = record.url + it[title] = record.title + it[thumbnailUrl] = record.thumbnailUrl + it[uploaderName] = record.uploaderName + it[streamCount] = record.streamCount + it[playlistType] = record.playlistType + it[savedAt] = record.savedAt + }.insertedCount +} + +private fun stableLibraryId(userId: String, value: String): String = UUID.nameUUIDFromBytes( + "$userId:saved:$value".toByteArray(StandardCharsets.UTF_8), +).toString() From ab778b3c92c41a527b9753d0418b4f330c866386 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:41 +0200 Subject: [PATCH 17/68] feat: encode TypeType library and settings records --- .../TypeTypePortabilityLibraryWriter.kt | 105 ++++++++++++++++++ .../TypeTypePortabilitySettingsExport.kt | 19 ++++ .../TypeTypePortabilitySettingsImport.kt | 17 +++ .../TypeTypePortabilitySettingsWriter.kt | 22 ++++ 4 files changed, 163 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt new file mode 100644 index 00000000..31d2946f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt @@ -0,0 +1,105 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityLibraryWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + writeVideos(json, source, categories, PortabilityCategory.WATCH_LATER, "watchLater") + writeVideos(json, source, categories, PortabilityCategory.FAVORITES, "favorites") + progress(json, source, categories) + searchHistory(json, source, categories) + savedPlaylists(json, source, categories) + } + + private fun writeVideos( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + category: PortabilityCategory, + field: String, + ) { + if (category !in categories) return + json.writeArrayFieldStart(field) + source.forEach(category) { record -> + val video = when (record) { + is PortabilityWatchLater -> record.video + is PortabilityFavorite -> record.video + else -> return@forEach + } + json.writeStartObject() + if (record is PortabilityFavorite) { + json.writeStringField("videoUrl", video.url) + json.writeNumberField("favoritedAt", record.favoritedAt) + } else { + json.writeStringField("url", video.url) + json.writeNumberField("addedAt", (record as PortabilityWatchLater).addedAt) + } + json.writeVideoFields(video) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun progress( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.PROGRESS !in categories) return + json.writeArrayFieldStart("progress") + source.forEach(PortabilityCategory.PROGRESS) { record -> + if (record !is PortabilityProgress) return@forEach + json.writeStartObject() + json.writeStringField("videoUrl", record.videoUrl) + json.writeNumberField("position", record.positionSeconds) + json.writeNumberField("updatedAt", record.updatedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + if (record !is PortabilitySearchHistory) return@forEach + json.writeStartObject() + json.writeStringField("term", record.term) + json.writeNumberField("searchedAt", record.searchedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun savedPlaylists( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SAVED_PLAYLISTS !in categories) return + json.writeArrayFieldStart("savedPlaylists") + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + if (record !is PortabilitySavedPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("id", record.sourceId) + json.writeStringField("publicPlaylistId", record.sourceId) + json.writeStringField("url", record.url) + json.writeStringField("title", record.title) + json.writeStringField("thumbnailUrl", record.thumbnailUrl) + json.writeStringField("uploaderName", record.uploaderName) + json.writeNumberField("streamCount", record.streamCount) + json.writeStringField("playlistType", record.playlistType) + json.writeNumberField("savedAt", record.savedAt) + json.writeEndObject() + } + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt new file mode 100644 index 00000000..f42d596b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.db.tables.SettingsTable +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.services.toSettingsItem +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.jsonObject +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilitySettingsExport { + fun write(userId: String, sink: PortabilityRecordSink) { + val settings = SettingsTable.selectAll().where { SettingsTable.userId eq userId } + .singleOrNull()?.toSettingsItem() ?: SettingsItem() + val json = CacheJson.parseToJsonElement(CacheJson.encodeToString(settings)).jsonObject + sink.write(PortabilitySettings(json)) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt new file mode 100644 index 00000000..9479bf65 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.services.TypeTypeBackupLibraryRestore + +internal object TypeTypePortabilitySettingsImport { + fun write(userId: String, source: PortabilityRecordSource): Long { + var count = 0L + source.forEach(PortabilityCategory.SETTINGS) { record -> + if (record !is PortabilitySettings) return@forEach + val settings = CacheJson.decodeFromString(record.values.toString()) + count += TypeTypeBackupLibraryRestore.settings(userId, settings) + } + return count + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt new file mode 100644 index 00000000..0ca1d0f6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt @@ -0,0 +1,22 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilitySettingsWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SETTINGS !in categories) return + json.writeFieldName("settings") + var written = false + source.forEach(PortabilityCategory.SETTINGS) { record -> + if (record is PortabilitySettings && !written) { + json.writeRawValue(record.values.toString()) + written = true + } + } + if (!written) json.writeNull() + } +} From 3259c8b4c6f9248e0eeb7c231906742581807247 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:41 +0200 Subject: [PATCH 18/68] feat: add the TypeType portability adapter --- .../portability/TypeTypePortabilityAdapter.kt | 69 ++++++++ .../portability/TypeTypePortabilityReader.kt | 159 ++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt new file mode 100644 index 00000000..6fec6714 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT +import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION +import java.io.OutputStream + +class TypeTypePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.TYPE_TYPE, + adapterVersion = 1, + capabilities = PortabilityCategory.entries.mapTo(linkedSetOf()) { category -> + PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.COMPLETE, + ) + }, + defaultExtension = "json", + contentType = "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (!probe.contains("\"format\":\"$TYPE_TYPE_BACKUP_FORMAT\"")) return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.TYPE_TYPE, version, 100, "TypeType backup marker") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions" -> sink.read(PortabilityCategory.SUBSCRIPTIONS) { TypeTypePortabilityReader.subscriptions(parser, token, sink) } + "subscriptionGroups" -> sink.read(PortabilityCategory.SUBSCRIPTION_GROUPS) { TypeTypePortabilityReader.groups(parser, token, sink) } + "history" -> sink.read(PortabilityCategory.HISTORY) { TypeTypePortabilityReader.history(parser, token, sink) } + "playlists" -> sink.read(PortabilityCategory.PLAYLISTS) { TypeTypePortabilityReader.playlists(parser, token, sink) } + "watchLater" -> sink.read(PortabilityCategory.WATCH_LATER) { TypeTypePortabilityReader.watchLater(parser, token, sink) } + "favorites" -> sink.read(PortabilityCategory.FAVORITES) { TypeTypePortabilityReader.favorites(parser, token, sink) } + "progress" -> sink.read(PortabilityCategory.PROGRESS) { TypeTypePortabilityReader.progress(parser, token, sink) } + "searchHistory" -> sink.read(PortabilityCategory.SEARCH_HISTORY) { TypeTypePortabilityReader.searchHistory(parser, token, sink) } + "savedPlaylists" -> sink.read(PortabilityCategory.SAVED_PLAYLISTS) { TypeTypePortabilityReader.savedPlaylists(parser, token, sink) } + "settings" -> sink.read(PortabilityCategory.SETTINGS) { TypeTypePortabilityReader.settings(parser, token, sink) } + "contentFilters" -> sink.read(PortabilityCategory.CONTENT_FILTERS) { TypeTypePortabilityReader.contentFilters(parser, token, sink) } + else -> parser.skipChildren() + } + } + require(format == TYPE_TYPE_BACKUP_FORMAT) { "Unsupported TypeType backup format" } + require(version == TYPE_TYPE_BACKUP_VERSION) { "Unsupported TypeType backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = TypeTypePortabilityWriter.write(source, output, categories) +} + +private inline fun PortabilityRecordSink.read(category: PortabilityCategory, block: () -> Unit) { + markCategory(category) + block() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt new file mode 100644 index 00000000..a655b449 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt @@ -0,0 +1,159 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.AllowedChannelItem +import dev.typetype.server.models.AllowedPlaylistItem +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.models.ProgressItem +import dev.typetype.server.models.SavedPlaylistItem +import dev.typetype.server.models.SearchHistoryItem +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionGroupBackupItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.TypeTypeContentFiltersBackup +import dev.typetype.server.models.WatchLaterItem +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.put + +internal object TypeTypePortabilityReader { + fun subscriptions(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilitySubscription(it.channelUrl, it.name, it.avatarUrl, it.subscribedAt)) } + + fun groups(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { group -> + sink.write(PortabilitySubscriptionGroup(group.name)) + group.channelUrls.forEach { sink.write(PortabilitySubscriptionGroupMembership(group.name, it)) } + } + + fun history(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityHistory( + PortabilityVideo( + item.url, + item.title, + item.thumbnail, + item.duration, + item.channelName, + item.channelUrl, + item.channelAvatar, + ), + item.watchedAt, + item.progress, + ), + ) + } + + fun playlists(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { playlist -> + val sourceId = playlist.id.ifBlank { "${playlist.name}:${playlist.createdAt}" } + sink.write(PortabilityPlaylist(sourceId, playlist.name, playlist.description, playlist.createdAt)) + playlist.videos.forEach { video -> + sink.write( + PortabilityPlaylistVideo( + sourceId, + video.position, + PortabilityVideo( + video.url, + video.title, + video.thumbnail, + video.duration, + video.channelName, + video.channelUrl, + video.channelAvatar, + video.viewCount, + video.publishedAt, + ), + video.addedAt, + ), + ) + } + } + + fun watchLater(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityWatchLater( + PortabilityVideo(item.url, item.title, item.thumbnail, item.duration, item.channelName, item.channelUrl, item.channelAvatar, item.viewCount, item.publishedAt), + item.addedAt, + ), + ) + } + + fun favorites(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityFavorite( + PortabilityVideo(item.videoUrl, item.title, item.thumbnail, item.duration, item.channelName, item.channelUrl, item.channelAvatar, item.viewCount, item.publishedAt), + item.favoritedAt, + ), + ) + } + + fun progress(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilityProgress(it.videoUrl, it.position, it.updatedAt)) } + + fun searchHistory(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilitySearchHistory(it.term, it.searchedAt)) } + + fun savedPlaylists(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { + sink.write(PortabilitySavedPlaylist(it.publicPlaylistId, it.url, it.title, it.thumbnailUrl, it.uploaderName, it.streamCount, it.playlistType, it.savedAt)) + } + + fun settings(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) { + if (token == JsonToken.VALUE_NULL) return + val item = parser.decodeCurrent() + sink.write(PortabilitySettings(CacheJson.parseToJsonElement(CacheJson.encodeToString(SettingsItem.serializer(), item)).jsonObject)) + } + + fun contentFilters(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) { + if (token == JsonToken.VALUE_NULL) return + val filters = parser.decodeCurrent() + filters.blockedChannels.forEach { sink.write(it.toFilter("blockedChannel")) } + filters.blockedVideos.forEach { sink.write(it.toFilter("blockedVideo")) } + filters.blockedKeywords.forEach { sink.write(PortabilityContentFilter("blockedKeyword", it.keyword, createdAt = it.blockedAt)) } + filters.allowedChannels.forEach { sink.write(it.toFilter()) } + filters.allowedPlaylists.forEach { sink.write(it.toFilter()) } + } +} + +private inline fun JsonParser.readArray(token: JsonToken, block: (T) -> Unit) { + if (token == JsonToken.VALUE_NULL) return + require(token == JsonToken.START_ARRAY) { "Invalid TypeType backup section" } + while (nextToken() != JsonToken.END_ARRAY) block(decodeCurrent()) +} + +private inline fun JsonParser.decodeCurrent(): T = + CacheJson.decodeFromString(readJsonElement().toString()) + +private fun BlockedItem.toFilter(kind: String) = PortabilityContentFilter( + kind, + url, + name.orEmpty(), + thumbnailUrl.orEmpty(), + blockedAt, +) + +private fun AllowedChannelItem.toFilter() = PortabilityContentFilter( + "allowedChannel", + url, + name.orEmpty(), + thumbnailUrl.orEmpty(), + allowedAt, +) + +private fun AllowedPlaylistItem.toFilter() = PortabilityContentFilter( + "allowedPlaylist", + url, + title.orEmpty(), + thumbnailUrl.orEmpty(), + allowedAt, + kotlinx.serialization.json.buildJsonObject { put("uploaderName", uploaderName.orEmpty()) }, +) From 66e34baa57fbc34c2fa298d2b3b2b4b466d1cf58 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:36:42 +0200 Subject: [PATCH 19/68] test: cover TypeType portability round trips --- .../TypeTypePortabilityAdapterTest.kt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt new file mode 100644 index 00000000..a50fa784 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class TypeTypePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `legacy TypeType backup round trips through canonical records`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://youtube.com/channel/UC1", "Channel", subscribedAt = 10)) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://youtube.com/channel/UC1")) + source.write(PortabilityPlaylist("playlist-1", "Saved", "Description", 20)) + source.write( + PortabilityPlaylistVideo( + "playlist-1", + 0, + PortabilityVideo("https://youtube.com/watch?v=video000001", "Video"), + 30, + ), + ) + source.write(PortabilitySettings(buildJsonObject { put("defaultQuality", "720p") })) + source.write(PortabilityContentFilter("blockedKeyword", "spoiler", createdAt = 40)) + + val adapter = TypeTypePortabilityAdapter() + val output = ByteArrayOutputStream() + adapter.encode(source, output, source.categories()) + val file = directory.resolve("typetype.json") + Files.write(file, output.toByteArray()) + val input = PortabilityInputFactory.create(file, "typetype.json", "application/json") + val restored = PortabilitySpool.create(directory) + + assertEquals(PortabilityFormat.TYPE_TYPE, adapter.detect(input)?.format) + adapter.decode(input, restored) + + assertEquals(source.counts(), restored.counts()) + assertTrue(output.toString().contains("\"subscriptionGroups\"")) + assertTrue(output.toString().contains("\"blockedKeywords\"")) + restored.delete() + source.delete() + } +} From c167a8d60d3d90506ff3d520d3aacc1c5aefa158 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:05 +0200 Subject: [PATCH 20/68] feat: add portability JSON and upload helpers --- .../portability/PortabilityJsonElements.kt | 31 +++++++++++++++++++ .../portability/PortabilityUploadWriter.kt | 27 ++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt new file mode 100644 index 00000000..a6d0dc0d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt @@ -0,0 +1,31 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive + +internal fun JsonObject.string(name: String): String = get(name)?.let { element -> + runCatching { element.jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") +}.orEmpty() + +internal fun JsonObject.long(name: String): Long = get(name)?.let { element -> + runCatching { element.jsonPrimitive.contentOrNull?.toLongOrNull() ?: 0L }.getOrDefault(0L) +} ?: 0L + +internal fun JsonObject.int(name: String): Int = long(name).toInt() + +internal fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> + runCatching { element.jsonArray }.getOrNull() +} ?: JsonArray(emptyList()) + +internal fun JsonElement.stringValue(): String = runCatching { + jsonPrimitive.contentOrNull.orEmpty() +}.getOrDefault("") + +internal fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject + +internal fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt new file mode 100644 index 00000000..624b0699 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.portability + +import io.ktor.utils.io.ByteReadChannel +import io.ktor.utils.io.jvm.javaio.toInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path + +object PortabilityUploadWriter { + suspend fun write(channel: ByteReadChannel, target: Path): Long = withContext(Dispatchers.IO) { + var written = 0L + channel.toInputStream().use { input -> + Files.newOutputStream(target).use { output -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read <= 0) break + written += read + if (written > PortabilityLimits.MAX_UPLOAD_BYTES) throw PortabilityUploadTooLargeException() + output.write(buffer, 0, read) + } + } + } + written + } +} From e21c223b9a6fd9d2da017b0d8f03733b7934e66c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:05 +0200 Subject: [PATCH 21/68] feat: add Invidious portability support --- .../InvidiousPortabilityAdapter.kt | 208 ++++++++++++++++++ .../InvidiousPortabilityAdapterTest.kt | 41 ++++ 2 files changed, 249 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt new file mode 100644 index 00000000..dc3b9dcc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt @@ -0,0 +1,208 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonObject +import java.io.OutputStream + +class InvidiousPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.INVIDIOUS, + adapterVersion = 1, + capabilities = setOf( + capability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + capability(PortabilityCategory.HISTORY, PortabilityFidelity.PARTIAL), + capability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + capability(PortabilityCategory.SETTINGS, PortabilityFidelity.PARTIAL), + ), + defaultExtension = "json", + contentType = "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + val hasHistory = probe.contains("\"watch_history\"") + val hasPreferences = probe.contains("\"preferences\"") + val hasSubscriptions = probe.contains("\"subscriptions\"") + if (!hasSubscriptions || (!hasHistory && !hasPreferences)) return null + return PortabilityDetection(PortabilityFormat.INVIDIOUS, null, 96, "Invidious account export fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when { + field == "subscriptions" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readSubscriptions(parser, sink) + } + field == "watch_history" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.HISTORY) + readHistory(parser, sink) + } + field == "playlists" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.PLAYLISTS) + readPlaylists(parser, sink) + } + field == "preferences" && token == JsonToken.START_OBJECT -> { + sink.markCategory(PortabilityCategory.SETTINGS) + sink.write(PortabilitySettings(JsonObject(mapOf("invidiousPreferences" to parser.readJsonElement())))) + } + else -> parser.skipChildren() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + writeSubscriptions(json, source, categories) + writeHistory(json, source, categories) + json.writeObjectFieldStart("preferences") + json.writeEndObject() + writePlaylists(json, source, categories) + json.writeEndObject() + } + } + + private fun readSubscriptions(parser: JsonParser, sink: PortabilityRecordSink) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + val channel = youtubeChannelUrl(parser.textOrEmpty()) + if (channel.isNotBlank()) sink.write(PortabilitySubscription(channel)) + } + } + + private fun readHistory(parser: JsonParser, sink: PortabilityRecordSink) { + var missingDates = 0L + while (parser.nextToken() != JsonToken.END_ARRAY) { + val url = youtubeVideoUrl(parser.textOrEmpty()) + if (url.isNotBlank()) { + sink.write(PortabilityHistory(PortabilityVideo(url), watchedAt = 0L)) + missingDates += 1 + } + } + if (missingDates > 0L) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "missing_history_dates", + "Invidious does not include original watch dates", + missingDates, + ), + ) + } + } + + private fun readPlaylists(parser: JsonParser, sink: PortabilityRecordSink) { + var index = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(parser.currentToken() == JsonToken.START_OBJECT) { "Invalid Invidious playlist" } + val playlist = readPlaylist(parser, index++) + sink.write(PortabilityPlaylist(playlist.id, playlist.title, playlist.description)) + playlist.videoIds.forEachIndexed { position, videoId -> + sink.write( + PortabilityPlaylistVideo( + playlist.id, + position, + PortabilityVideo(youtubeVideoUrl(videoId)), + ), + ) + } + } + } + + private fun readPlaylist(parser: JsonParser, index: Int): ParsedPlaylist { + var title = "" + var description = "" + val videoIds = mutableListOf() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when { + field == "title" -> title = parser.textOrEmpty() + field == "description" -> description = parser.textOrEmpty() + field == "videos" && token == JsonToken.START_ARRAY -> { + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(videoIds.size < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Playlist contains too many videos" } + videoIds += parser.textOrEmpty() + } + } + else -> parser.skipChildren() + } + } + val id = "invidious:$index:${title.trim().lowercase()}" + return ParsedPlaylist(id, title, description, videoIds) + } + + private fun writeSubscriptions( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("subscriptions") + if (PortabilityCategory.SUBSCRIPTIONS in categories) { + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + json.writeString(youtubeId((record as PortabilitySubscription).channelUrl)) + } + } + json.writeEndArray() + } + + private fun writeHistory( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("watch_history") + if (PortabilityCategory.HISTORY in categories) { + source.forEach(PortabilityCategory.HISTORY) { record -> + json.writeString(youtubeId((record as PortabilityHistory).video.url)) + } + } + json.writeEndArray() + } + + private fun writePlaylists( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("playlists") + if (PortabilityCategory.PLAYLISTS in categories) { + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("title", record.name) + json.writeStringField("description", record.description) + json.writeStringField("privacy", "Private") + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + if (child is PortabilityPlaylistVideo) json.writeString(youtubeId(child.video.url)) + } + json.writeEndArray() + json.writeEndObject() + } + } + json.writeEndArray() + } + + private data class ParsedPlaylist( + val id: String, + val title: String, + val description: String, + val videoIds: List, + ) +} + +private fun capability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt new file mode 100644 index 00000000..cba31de5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class InvidiousPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves invidious account categories`() { + val file = directory.resolve("invidious.json") + Files.writeString( + file, + """{"subscriptions":["UC1"],"watch_history":["video000001"],"preferences":{"thin_mode":true},"playlists":[{"title":"Saved","description":"Keep","privacy":"Private","videos":["video000002"]}]}""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = InvidiousPortabilityAdapter() + + assertEquals(PortabilityFormat.INVIDIOUS, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.SETTINGS]) + assertEquals("missing_history_dates", spool.issues().single().code) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + val json = output.toString() + assertTrue(json.contains("\"subscriptions\":[\"UC1\"]")) + assertTrue(json.contains("\"title\":\"Saved\"")) + spool.delete() + } +} From d05beb0dc308d2549751fff94f68d17347706414 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:06 +0200 Subject: [PATCH 22/68] feat: decode Piped account backups --- .../portability/PipedPortabilityAdapter.kt | 82 +++++++++++++++++ .../portability/PipedPortabilityReader.kt | 90 +++++++++++++++++++ .../PipedPortabilityAdapterTest.kt | 47 ++++++++++ 3 files changed, 219 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt new file mode 100644 index 00000000..02b3e783 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class PipedPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.PIPED, + 1, + setOf( + pipedCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.HISTORY, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (LIBRE_TUBE_FIELDS.any(probe::contains)) return null + if (probe.contains("\"format\"") && probe.contains("\"Piped\"")) { + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.PIPED, version, 96, "Piped format marker") + } + val trimmed = probe.trimStart() + if (trimmed.startsWith("[") && (probe.contains("UC") || probe.contains("youtube.com/channel/"))) { + return PortabilityDetection(PortabilityFormat.PIPED, null, 70, "Piped subscription array") + } + return null + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + when (parser.nextToken()) { + JsonToken.START_ARRAY -> PipedPortabilityReader.subscriptions(parser, sink) + JsonToken.START_OBJECT -> readObject(parser, sink) + else -> error("Invalid Piped backup") + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = PipedPortabilityWriter.write(source, output, categories) + + private fun readObject(parser: com.fasterxml.jackson.core.JsonParser, sink: PortabilityRecordSink) { + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions", "localSubscriptions" -> PipedPortabilityReader.subscriptions(parser, sink, token) + "groups", "channelGroups" -> PipedPortabilityReader.groups(parser, sink, token) + "watchHistory" -> PipedPortabilityReader.history(parser, sink, token) + "playlists" -> PipedPortabilityReader.playlists(parser, sink, token) + else -> parser.skipChildren() + } + } + require(format == "Piped" && version == 1) { "Unsupported Piped backup version" } + } + + private companion object { + val LIBRE_TUBE_FIELDS = listOf( + "\"watchPositions\"", + "\"customInstances\"", + "\"playlistBookmarks\"", + "\"localPlaylists\"", + ) + } +} + +private fun pipedCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt new file mode 100644 index 00000000..30703bf9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject + +internal object PipedPortabilityReader { + fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken = JsonToken.START_ARRAY) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped subscriptions" } + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readArray(parser) { element -> + val item = element as? JsonObject + val channel = item?.string("channelId") ?: item?.string("url") ?: element.stringValue() + if (channel.isNotBlank()) { + sink.write( + PortabilitySubscription( + youtubeChannelUrl(channel), + item?.string("name") ?: item?.string("channelName").orEmpty(), + item?.string("avatar") ?: item?.string("channelThumbnail").orEmpty(), + item?.long("subscribedAt") ?: 0L, + ), + ) + } + } + } + + fun groups(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped groups" } + sink.markCategory(PortabilityCategory.SUBSCRIPTION_GROUPS) + readArray(parser) { element -> + val item = element.jsonObject + val name = item.string("groupName").ifBlank { item.string("name") } + if (name.isBlank()) return@readArray + sink.write(PortabilitySubscriptionGroup(name)) + item.array("channels").forEach { channel -> + val value = channel.stringValue() + if (value.isNotBlank()) sink.write(PortabilitySubscriptionGroupMembership(name, youtubeChannelUrl(value))) + } + } + } + + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped history" } + sink.markCategory(PortabilityCategory.HISTORY) + readArray(parser) { element -> + val item = element.jsonObject + val videoId = item.string("videoId") + if (videoId.isBlank()) return@readArray + sink.write( + PortabilityHistory( + PortabilityVideo( + youtubeVideoUrl(videoId), + item.string("title"), + item.string("thumbnail"), + item.long("duration"), + item.string("uploaderName"), + item.string("uploaderUrl"), + ), + item.long("watchedAt"), + item.long("currentTime"), + ), + ) + } + } + + fun playlists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped playlists" } + sink.markCategory(PortabilityCategory.PLAYLISTS) + var index = 0 + readArray(parser) { element -> + val item = element.jsonObject + val name = item.string("name") + val sourceId = "piped:${index++}:${name.lowercase()}" + sink.write(PortabilityPlaylist(sourceId, name)) + item.array("videos").forEachIndexed { position, video -> + sink.write(PortabilityPlaylistVideo(sourceId, position, PortabilityVideo(youtubeVideoUrl(video.stringValue())))) + } + } + } + + private fun readArray(parser: JsonParser, block: (JsonElement) -> Unit) { + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement()) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt new file mode 100644 index 00000000..d92af2b6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt @@ -0,0 +1,47 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class PipedPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves piped account categories`() { + val file = directory.resolve("piped.json") + Files.writeString( + file, + """ + { + "format":"Piped","version":1, + "subscriptions":["UC1"], + "groups":[{"groupName":"News","channels":["UC1"]}], + "watchHistory":[{"videoId":"video1","title":"One","watchedAt":12,"currentTime":4}], + "playlists":[{"name":"Saved","videos":["https://youtube.com/watch?v=video1"]}] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = PipedPortabilityAdapter() + + assertEquals(PortabilityFormat.PIPED, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + assertTrue(output.toString().contains("\"format\":\"Piped\"")) + assertTrue(output.toString().contains("\"groupName\":\"News\"")) + spool.delete() + } +} From dc08fa3f83a87fca047342acb5d4d18fdd23642b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:06 +0200 Subject: [PATCH 23/68] feat: encode Piped account backups --- .../portability/PipedPortabilityWriter.kt | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt new file mode 100644 index 00000000..304e6cde --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt @@ -0,0 +1,83 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object PipedPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", "Piped") + json.writeNumberField("version", 1) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + playlists(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + json.writeString(youtubeId((record as PortabilitySubscription).channelUrl)) + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("groups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("groupName", record.name) + json.writeArrayFieldStart("channels") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + json.writeString(youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl)) + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("watchHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("uploaderName", item.video.channelName) + json.writeStringField("uploaderUrl", item.video.channelUrl) + json.writeStringField("thumbnail", item.video.thumbnailUrl) + json.writeNumberField("duration", item.video.durationSeconds) + json.writeNumberField("watchedAt", item.watchedAt) + json.writeNumberField("currentTime", item.positionSeconds) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun playlists(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeStringField("type", "playlist") + json.writeStringField("visibility", "private") + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + json.writeString((child as PortabilityPlaylistVideo).video.url) + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } +} From 6138a2c90161f16ccdcc994a9b5ee5780963781a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:07 +0200 Subject: [PATCH 24/68] feat: decode LibreTube account backups --- .../LibreTubePortabilityAdapter.kt | 76 +++++++++++++ .../portability/LibreTubePortabilityReader.kt | 106 ++++++++++++++++++ .../LibreTubePortabilityAdapterTest.kt | 49 ++++++++ 3 files changed, 231 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt new file mode 100644 index 00000000..d755c94a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt @@ -0,0 +1,76 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class LibreTubePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.LIBRE_TUBE, + 1, + setOf( + libreCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.HISTORY, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.PROGRESS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.SEARCH_HISTORY, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.SAVED_PLAYLISTS, PortabilityFidelity.COMPLETE), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (!probe.contains("\"format\"") || !probe.contains("\"Piped\"")) return null + val evidence = UNIQUE_FIELDS.firstOrNull(probe::contains) ?: return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.LIBRE_TUBE, version, 99, "LibreTube field $evidence") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions", "localSubscriptions" -> PipedPortabilityReader.subscriptions(parser, sink, token) + "groups", "channelGroups" -> PipedPortabilityReader.groups(parser, sink, token) + "watchHistory" -> LibreTubePortabilityReader.history(parser, sink, token) + "watchPositions" -> LibreTubePortabilityReader.positions(parser, sink, token) + "searchHistory" -> LibreTubePortabilityReader.searchHistory(parser, sink, token) + "localPlaylists" -> LibreTubePortabilityReader.localPlaylists(parser, sink, token) + "playlistBookmarks" -> LibreTubePortabilityReader.bookmarks(parser, sink, token) + else -> parser.skipChildren() + } + } + require(format == "Piped" && version == 1) { "Unsupported LibreTube backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = LibreTubePortabilityWriter.write(source, output, categories) + + private companion object { + val UNIQUE_FIELDS = listOf( + "\"watchPositions\"", + "\"customInstances\"", + "\"playlistBookmarks\"", + "\"localPlaylists\"", + "\"preferences\"", + ) + } +} + +private fun libreCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt new file mode 100644 index 00000000..afb7c4f2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt @@ -0,0 +1,106 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonObject + +internal object LibreTubePortabilityReader { + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.HISTORY, parser, token) { element -> + val item = element.jsonObject + val id = item.string("videoId") + if (id.isBlank()) return@readArray + sink.write( + PortabilityHistory( + PortabilityVideo( + youtubeVideoUrl(id), + item.string("title"), + item.string("thumbnailUrl"), + item.long("duration"), + item.string("uploader"), + item.string("uploaderUrl"), + item.string("uploaderAvatar"), + ), + 0L, + ), + ) + } + } + + fun positions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.PROGRESS, parser, token) { element -> + val item = element.jsonObject + val id = item.string("videoId") + if (id.isNotBlank()) sink.write(PortabilityProgress(youtubeVideoUrl(id), item.long("position") / 1_000L)) + } + } + + fun searchHistory(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.SEARCH_HISTORY, parser, token) { element -> + val query = element.jsonObject.string("query") + if (query.isNotBlank()) sink.write(PortabilitySearchHistory(query, 0L)) + } + } + + fun localPlaylists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.PLAYLISTS, parser, token) { element -> + val item = element.jsonObject + val playlist = item["playlist"]?.objectOrNull() ?: return@readArray + val sourceId = "libretube:${playlist.string("id")}:${playlist.string("name").lowercase()}" + sink.write(PortabilityPlaylist(sourceId, playlist.string("name"), playlist.string("description"))) + item.array("videos").forEachIndexed { position, videoElement -> + val video = videoElement.jsonObject + sink.write( + PortabilityPlaylistVideo( + sourceId, + position, + PortabilityVideo( + youtubeVideoUrl(video.string("videoId")), + video.string("title"), + video.string("thumbnailUrl"), + video.long("duration"), + video.string("uploader"), + video.string("uploaderUrl"), + video.string("uploaderAvatar"), + ), + ), + ) + } + } + } + + fun bookmarks(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.SAVED_PLAYLISTS, parser, token) { element -> + val item = element.jsonObject + val id = item.string("playlistId") + if (id.isNotBlank()) { + sink.write( + PortabilitySavedPlaylist( + id, + "https://www.youtube.com/playlist?list=$id", + item.string("playlistName"), + item.string("thumbnailUrl"), + item.string("uploader"), + item.long("videos"), + ), + ) + } + } + } +} + +private fun PortabilityRecordSink.readArray( + category: PortabilityCategory, + parser: JsonParser, + token: JsonToken, + block: (JsonElement) -> Unit, +) { + require(token == JsonToken.START_ARRAY) { "Invalid LibreTube backup section" } + markCategory(category) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement()) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt new file mode 100644 index 00000000..46eca457 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class LibreTubePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves libretube account data`() { + val file = directory.resolve("libretube.json") + Files.writeString( + file, + """ + { + "format":"Piped","version":1, + "localSubscriptions":[{"channelId":"UC1","name":"One","avatar":"avatar"}], + "watchHistory":[{"videoId":"v1","title":"Video","duration":60}], + "watchPositions":[{"videoId":"v1","position":12000}], + "searchHistory":[{"query":"query"}], + "playlistBookmarks":[{"playlistId":"PL1","playlistName":"Saved","videos":2}], + "localPlaylists":[{"playlist":{"id":1,"name":"Local"},"videos":[{"videoId":"v1","title":"Video"}]}] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = LibreTubePortabilityAdapter() + + assertEquals(PortabilityFormat.LIBRE_TUBE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + assertTrue(output.toString().contains("\"localSubscriptions\"")) + assertTrue(output.toString().contains("\"watchPositions\"")) + spool.delete() + } +} From 591c9972740e46d0c243770ce36e18d1e33323d3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:07 +0200 Subject: [PATCH 25/68] feat: encode LibreTube account backups --- .../portability/LibreTubePortabilityWriter.kt | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt new file mode 100644 index 00000000..f3eb1915 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt @@ -0,0 +1,145 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object LibreTubePortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", "Piped") + json.writeNumberField("version", 1) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + positions(json, source, categories) + searchHistory(json, source, categories) + localPlaylists(json, source, categories) + bookmarks(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("localSubscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + json.writeStartObject() + json.writeStringField("channelId", youtubeId(item.channelUrl)) + json.writeStringField("name", item.name) + json.writeStringField("avatar", item.avatarUrl) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("channelGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("groupName", record.name) + json.writeArrayFieldStart("channels") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + json.writeString(youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl)) + } + json.writeEndArray() + json.writeNumberField("index", 0) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("watchHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("uploader", item.video.channelName) + json.writeStringField("uploaderUrl", item.video.channelUrl) + json.writeStringField("uploaderAvatar", item.video.channelAvatarUrl) + json.writeStringField("thumbnailUrl", item.video.thumbnailUrl) + json.writeNumberField("duration", item.video.durationSeconds) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun positions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PROGRESS !in categories) return + json.writeArrayFieldStart("watchPositions") + source.forEach(PortabilityCategory.PROGRESS) { record -> + val item = record as PortabilityProgress + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.videoUrl)) + json.writeNumberField("position", item.positionSeconds * 1_000L) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + json.writeStartObject() + json.writeStringField("query", (record as PortabilitySearchHistory).term) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun localPlaylists(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("localPlaylists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeObjectFieldStart("playlist") + json.writeNumberField("id", 0) + json.writeStringField("name", record.name) + json.writeStringField("description", record.description) + json.writeStringField("thumbnailUrl", "") + json.writeEndObject() + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + val video = (child as PortabilityPlaylistVideo).video + json.writeStartObject() + json.writeNumberField("id", 0) + json.writeNumberField("playlistId", 0) + json.writeStringField("videoId", youtubeId(video.url)) + json.writeStringField("title", video.title) + json.writeStringField("thumbnailUrl", video.thumbnailUrl) + json.writeNumberField("duration", video.durationSeconds) + json.writeStringField("uploader", video.channelName) + json.writeStringField("uploaderUrl", video.channelUrl) + json.writeStringField("uploaderAvatar", video.channelAvatarUrl) + json.writeEndObject() + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } + + private fun bookmarks(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SAVED_PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlistBookmarks") + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + val item = record as PortabilitySavedPlaylist + json.writeStartObject() + json.writeStringField("playlistId", item.sourceId) + json.writeStringField("playlistName", item.title) + json.writeStringField("thumbnailUrl", item.thumbnailUrl) + json.writeStringField("uploader", item.uploaderName) + json.writeNumberField("videos", item.streamCount) + json.writeEndObject() + } + json.writeEndArray() + } +} From 3f60087247f9cc62b12324de2dc4a633d66a2286 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:07 +0200 Subject: [PATCH 26/68] feat: decode Flow account backups --- .../portability/FlowPortabilityAdapter.kt | 69 ++++++++++ .../portability/FlowPortabilityReader.kt | 119 ++++++++++++++++++ .../portability/FlowPortabilityAdapterTest.kt | 54 ++++++++ 3 files changed, 242 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt new file mode 100644 index 00000000..1fa4b033 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class FlowPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.FLOW, + 2, + setOf( + flowCapability(PortabilityCategory.SUBSCRIPTIONS), + flowCapability(PortabilityCategory.SUBSCRIPTION_GROUPS), + flowCapability(PortabilityCategory.HISTORY), + flowCapability(PortabilityCategory.PROGRESS), + flowCapability(PortabilityCategory.SEARCH_HISTORY), + flowCapability(PortabilityCategory.FAVORITES), + flowCapability(PortabilityCategory.CONTENT_FILTERS, PortabilityFidelity.PARTIAL), + PortabilityCapability(PortabilityCategory.PLAYLISTS, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.PARTIAL), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (!probe.contains("\"viewHistory\"") || !probe.contains("\"subscriptionGroups\"")) return null + if (!probe.contains("\"playlistVideos\"") && !probe.contains("\"likedVideos\"")) return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.FLOW, version, 98, "Flow backup fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "version" -> version = parser.intValue + "subscriptions" -> FlowPortabilityReader.subscriptions(parser, sink, token) + "subscriptionGroups" -> FlowPortabilityReader.groups(parser, sink, token) + "viewHistory" -> FlowPortabilityReader.history(parser, sink, token) + "searchHistory" -> FlowPortabilityReader.searchHistory(parser, sink, token) + "playlists" -> FlowPortabilityReader.playlists(parser, sink, token) + "playlistVideos" -> FlowPortabilityReader.playlistVideos(parser, sink, token) + "likedVideos" -> FlowPortabilityReader.favorites(parser, sink, token) + "contentPreferences" -> FlowPortabilityReader.contentFilters(parser, sink, token) + else -> parser.skipChildren() + } + } + require(version == 2) { "Unsupported Flow backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = FlowPortabilityWriter.write(source, output, categories) +} + +private fun flowCapability( + category: PortabilityCategory, + fidelity: PortabilityFidelity = PortabilityFidelity.COMPLETE, +) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt new file mode 100644 index 00000000..ecc999e6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt @@ -0,0 +1,119 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.jsonObject + +internal object FlowPortabilityReader { + fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SUBSCRIPTIONS, parser, token) { item -> + val id = item.string("channelId") + if (id.isNotBlank()) { + sink.write( + PortabilitySubscription( + youtubeChannelUrl(id), + item.string("channelName"), + item.string("channelThumbnail"), + item.long("subscribedAt"), + ), + ) + } + } + + fun groups(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SUBSCRIPTION_GROUPS, parser, token) { item -> + val name = item.string("name") + if (name.isBlank()) return@readFlowArray + sink.write(PortabilitySubscriptionGroup(name)) + var memberships = 0 + item.string("channelIds").splitToSequence(',').map(String::trim).filter(String::isNotBlank).forEach { channel -> + require(memberships++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Flow group contains too many channels" } + sink.write(PortabilitySubscriptionGroupMembership(name, youtubeChannelUrl(channel))) + } + } + + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.markCategory(PortabilityCategory.PROGRESS) + sink.readFlowArray(PortabilityCategory.HISTORY, parser, token) { item -> + val id = item.string("videoId") + if (id.isBlank()) return@readFlowArray + val video = PortabilityVideo( + youtubeVideoUrl(id), + item.string("title"), + item.string("thumbnailUrl"), + item.long("duration") / 1_000L, + item.string("channelName"), + youtubeChannelUrl(item.string("channelId")), + ) + sink.write(PortabilityHistory(video, item.long("timestamp"), item.long("position") / 1_000L)) + sink.write(PortabilityProgress(video.url, item.long("position") / 1_000L, item.long("timestamp"))) + } + } + + fun searchHistory(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SEARCH_HISTORY, parser, token) { item -> + val query = item.string("query") + if (query.isNotBlank()) sink.write(PortabilitySearchHistory(query, item.long("timestamp"))) + } + + fun playlists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.PLAYLISTS, parser, token) { item -> + val id = item.string("id") + if (id.isNotBlank()) sink.write(PortabilityPlaylist(id, item.string("name"), item.string("description"), item.long("createdAt"))) + } + + fun playlistVideos(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.PLAYLISTS, parser, token) { item -> + val playlistId = item.string("playlistId") + val videoId = item.string("videoId") + if (playlistId.isNotBlank() && videoId.isNotBlank()) { + sink.write( + PortabilityPlaylistVideo( + playlistId, + item.long("position").coerceIn(0L, Int.MAX_VALUE.toLong()).toInt(), + PortabilityVideo(youtubeVideoUrl(videoId)), + item.long("addedAt"), + ), + ) + } + } + + fun favorites(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.FAVORITES, parser, token) { item -> + val id = item.string("videoId") + if (id.isNotBlank()) { + sink.write( + PortabilityFavorite( + PortabilityVideo(youtubeVideoUrl(id), item.string("title"), item.string("thumbnail"), channelName = item.string("channelName")), + item.long("likedAt"), + ), + ) + } + } + + fun contentFilters(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_OBJECT) { "Invalid Flow content preferences" } + sink.markCategory(PortabilityCategory.CONTENT_FILTERS) + val item = parser.readJsonElement().jsonObject + item.array("blockedChannels").forEach { channel -> + val value = channel.stringValue() + if (value.isNotBlank()) sink.write(PortabilityContentFilter("blockedChannel", youtubeChannelUrl(value))) + } + } +} + +private fun PortabilityRecordSink.readFlowArray( + category: PortabilityCategory, + parser: JsonParser, + token: JsonToken, + block: (kotlinx.serialization.json.JsonObject) -> Unit, +) { + if (token == JsonToken.VALUE_NULL) return + require(token == JsonToken.START_ARRAY) { "Invalid Flow backup section" } + markCategory(category) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement().jsonObject) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt new file mode 100644 index 00000000..dfd5541e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt @@ -0,0 +1,54 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class FlowPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves supported flow version two data`() { + val file = directory.resolve("flow.json") + Files.writeString( + file, + """ + { + "version":2,"timestamp":123, + "viewHistory":[{"videoId":"v1","position":12000,"duration":60000,"timestamp":10,"title":"One","channelId":"UC1"}], + "searchHistory":[{"query":"query","timestamp":11,"type":"TEXT"}], + "subscriptions":[{"channelId":"UC1","channelName":"Channel","channelThumbnail":"avatar","subscribedAt":12}], + "playlists":[{"id":"p1","name":"Saved","description":"List","createdAt":13}], + "playlistVideos":[{"playlistId":"p1","videoId":"v1","position":0,"addedAt":14}], + "videos":[{"id":"v1","title":"One","channelId":"UC1"}], + "subscriptionGroups":[{"name":"News","channelIds":"UC1","sortOrder":0}], + "likedVideos":[{"videoId":"v1","title":"One","likedAt":15}], + "contentPreferences":{"blockedChannels":["UC2"],"preferredTopics":[],"blockedTopics":[]} + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = FlowPortabilityAdapter() + + assertEquals(PortabilityFormat.FLOW, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories() - PortabilityCategory.PLAYLISTS) + assertTrue(output.toString().contains("\"version\":2")) + assertTrue(output.toString().contains("\"channelIds\":\"UC1\"")) + spool.delete() + } +} From 4f1c9e8c4f132523978a832b1cf536119218dc4c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:08 +0200 Subject: [PATCH 27/68] feat: encode Flow account backups --- .../portability/FlowPortabilityWriter.kt | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt new file mode 100644 index 00000000..f087b3bf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt @@ -0,0 +1,132 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object FlowPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeNumberField("version", 2) + json.writeNumberField("timestamp", System.currentTimeMillis()) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + searchHistory(json, source, categories) + favorites(json, source, categories) + filters(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + json.writeStartObject() + json.writeStringField("channelId", youtubeId(item.channelUrl)) + json.writeStringField("channelName", item.name) + json.writeStringField("channelThumbnail", item.avatarUrl) + json.writeNumberField("subscribedAt", item.subscribedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptionGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val channels = boundedChannelIds(source, record.name) + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeStringField("channelIds", channels) + json.writeNumberField("sortOrder", 0) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun boundedChannelIds(source: PortabilityRecordSource, groupName: String): String { + val channels = StringBuilder() + var count = 0 + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, groupName) { child -> + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Flow group contains too many channels" } + val channel = youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl) + require(channels.length + channel.length + 1 <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { + "Flow group channel list is too large" + } + if (channels.isNotEmpty()) channels.append(',') + channels.append(channel) + } + return channels.toString() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("viewHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeNumberField("position", item.positionSeconds * 1_000L) + json.writeNumberField("duration", item.video.durationSeconds * 1_000L) + json.writeNumberField("timestamp", item.watchedAt) + json.writeStringField("title", item.video.title) + json.writeStringField("thumbnailUrl", item.video.thumbnailUrl) + json.writeStringField("channelName", item.video.channelName) + json.writeStringField("channelId", youtubeId(item.video.channelUrl)) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + val item = record as PortabilitySearchHistory + json.writeStartObject() + json.writeStringField("query", item.term) + json.writeNumberField("timestamp", item.searchedAt) + json.writeStringField("type", "TEXT") + json.writeEndObject() + } + json.writeEndArray() + } + + private fun favorites(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.FAVORITES !in categories) return + json.writeArrayFieldStart("likedVideos") + source.forEach(PortabilityCategory.FAVORITES) { record -> + val item = record as PortabilityFavorite + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("thumbnail", item.video.thumbnailUrl) + json.writeStringField("channelName", item.video.channelName) + json.writeNumberField("likedAt", item.favoritedAt) + json.writeBooleanField("isMusic", false) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun filters(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.CONTENT_FILTERS !in categories) return + json.writeObjectFieldStart("contentPreferences") + json.writeArrayFieldStart("blockedChannels") + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + val item = record as PortabilityContentFilter + if (item.kind == "blockedChannel") json.writeString(youtubeId(item.value)) + } + json.writeEndArray() + json.writeArrayFieldStart("preferredTopics") + json.writeEndArray() + json.writeArrayFieldStart("blockedTopics") + json.writeEndArray() + json.writeEndObject() + } +} From 0a79f5f3276066196899e2ea06d2b92bb1b8d9aa Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:21 +0200 Subject: [PATCH 28/68] feat: decode Grayjay account backups --- .../portability/GrayjayPortabilityAdapter.kt | 65 +++++++++++ .../portability/GrayjayPortabilityReader.kt | 101 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt new file mode 100644 index 00000000..14234e24 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt @@ -0,0 +1,65 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class GrayjayPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.GRAYJAY, + 1, + setOf( + grayjayCapability(PortabilityCategory.SUBSCRIPTIONS, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, BOTH, PortabilityFidelity.COMPLETE), + grayjayCapability(PortabilityCategory.HISTORY, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.PROGRESS, IMPORT, PortabilityFidelity.COMPLETE), + grayjayCapability(PortabilityCategory.PLAYLISTS, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.WATCH_LATER, BOTH, PortabilityFidelity.PARTIAL), + ), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + if ("exportInfo" !in archive.names || archive.names.none { it.startsWith("stores/") }) return null + val version = ZipFile(input.path.toFile()).use { zip -> + val entry = zip.getEntry("exportInfo") ?: return null + zip.getInputStream(entry).buffered().use { stream -> + Regex("\"version\"\\s*:\\s*\"([^\"]+)\"") + .find(stream.readNBytes(PortabilityLimits.PROBE_BYTES).decodeToString()) + ?.groupValues?.get(1) + } + } + if (version != "1") return null + return PortabilityDetection(PortabilityFormat.GRAYJAY, version, 99, "Grayjay exportInfo and stores") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported or encrypted Grayjay backup" } + ZipFile(input.path.toFile()).use { zip -> GrayjayPortabilityReader.read(zip, sink) } + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = super.assessExport(source, categories) + categories.mapNotNull { category -> + val fidelity = descriptor.capabilities.firstOrNull { it.category == category }?.fidelity + if (fidelity != PortabilityFidelity.PARTIAL) return@mapNotNull null + PortabilityIssue(category, "grayjay_partial_metadata", "Grayjay reconstructs this category from URLs, so some TypeType metadata is not represented") + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = GrayjayPortabilityWriter.write(source, output, categories) +} + +private val IMPORT = setOf(PortabilityDirection.IMPORT) +private val BOTH = setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT) + +private fun grayjayCapability( + category: PortabilityCategory, + directions: Set, + fidelity: PortabilityFidelity, +) = PortabilityCapability(category, directions, fidelity) diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt new file mode 100644 index 00000000..29fba1d8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt @@ -0,0 +1,101 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import java.util.zip.ZipFile + +internal object GrayjayPortabilityReader { + fun read(zip: ZipFile, sink: PortabilityRecordSink) { + readStrings(zip, "stores/subscriptions") { readSubscription(it, sink) } + readStrings(zip, "stores/subscription_groups") { readGroup(it, sink) } + readStrings(zip, "stores/history") { readHistory(it, sink) } + readStrings(zip, "stores/playlists") { readPlaylist(it, sink) } + readStrings(zip, "stores/watch_later") { readWatchLater(it, sink) } + } + + private fun readStrings(zip: ZipFile, name: String, consume: (String) -> Unit) { + val entry = zip.getEntry(name) ?: return + zip.getInputStream(entry).buffered().use { input -> + PortabilityJsonFactory.createParser(input).use { parser -> + require(parser.nextToken() == JsonToken.START_ARRAY) { "$name must contain an array" } + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "$name contains too many records" } + require(parser.currentToken() == JsonToken.VALUE_STRING) { "$name contains an invalid record" } + consume(parser.text) + } + } + } + } + + private fun readSubscription(value: String, sink: PortabilityRecordSink) { + val url = value.trim() + if (url.isNotBlank()) sink.write(PortabilitySubscription(url)) + } + + private fun readWatchLater(value: String, sink: PortabilityRecordSink) { + val url = value.trim() + if (url.isNotBlank()) sink.write(PortabilityWatchLater(PortabilityVideo(url))) + } + + private fun readHistory(value: String, sink: PortabilityRecordSink) { + val parts = value.split("|||", limit = 4) + if (parts.size != 4) { + sink.issue(PortabilityIssue(PortabilityCategory.HISTORY, "grayjay_history_invalid", "A Grayjay history record could not be parsed")) + return + } + val url = parts[0].trim() + val watchedAt = parts[1].toLongOrNull() ?: 0L + val position = parts[2].toLongOrNull() ?: 0L + if (url.isBlank()) return + sink.write(PortabilityHistory(PortabilityVideo(url, title = parts[3]), watchedAt, position)) + sink.write(PortabilityProgress(url, position, watchedAt)) + } + + private fun readPlaylist(value: String, sink: PortabilityRecordSink) { + val lines = value.lineSequence().filter(String::isNotBlank).toList() + if (lines.isEmpty()) return + val header = lines.first() + val separator = header.indexOf(":::") + val name = if (separator >= 0) header.substring(0, separator) else header + val sourceId = if (separator >= 0) header.substring(separator + 3).ifBlank { name } else name + sink.write(PortabilityPlaylist(sourceId, name)) + lines.drop(1).forEachIndexed { index, url -> + sink.write(PortabilityPlaylistVideo(sourceId, index, PortabilityVideo(url.trim()))) + } + } + + private fun readGroup(value: String, sink: PortabilityRecordSink) { + PortabilityJsonFactory.createParser(value).use { parser -> readGroupObject(parser, sink) } + } + + private fun readGroupObject(parser: JsonParser, sink: PortabilityRecordSink) { + parser.requireObject() + var name = "" + val urls = ArrayList() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "name" -> name = parser.textOrEmpty() + "urls" -> readUrlArray(parser, token, urls) + else -> parser.skipChildren() + } + } + if (name.isBlank()) return + sink.write(PortabilitySubscriptionGroup(name)) + urls.forEach { sink.write(PortabilitySubscriptionGroupMembership(name, it)) } + } + + private fun readUrlArray(parser: JsonParser, token: JsonToken, urls: MutableList) { + if (token != JsonToken.START_ARRAY) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(urls.size < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Grayjay group contains too many channels" } + if (parser.currentToken() == JsonToken.VALUE_STRING && parser.text.isNotBlank()) urls += parser.text + else parser.skipChildren() + } + } +} From 95ea826b02a4590305ac3f67133435baeec9b917 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:22 +0200 Subject: [PATCH 29/68] feat: encode and verify Grayjay account backups --- .../portability/GrayjayPortabilityWriter.kt | 110 ++++++++++++++++++ .../GrayjayPortabilityAdapterTest.kt | 81 +++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt new file mode 100644 index 00000000..da23f917 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream +import java.util.UUID +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +internal object GrayjayPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + ZipOutputStream(output.buffered()).use { zip -> + writeObjectEntry(zip, "exportInfo", mapOf("version" to "1")) + if (PortabilityCategory.SUBSCRIPTIONS in categories) { + writeStore(zip, "subscriptions") { json -> + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + (record as? PortabilitySubscription)?.let { json.writeString(it.channelUrl) } + } + } + } + if (PortabilityCategory.SUBSCRIPTION_GROUPS in categories) writeGroups(zip, source) + if (PortabilityCategory.HISTORY in categories) writeHistory(zip, source) + if (PortabilityCategory.PLAYLISTS in categories) writePlaylists(zip, source) + if (PortabilityCategory.WATCH_LATER in categories) writeWatchLater(zip, source) + writeObjectEntry(zip, "plugins", emptyMap()) + writeObjectEntry(zip, "plugin_settings", emptyMap()) + } + } + + private fun writeHistory(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "history") { json -> + source.forEach(PortabilityCategory.HISTORY) { record -> + val history = record as? PortabilityHistory ?: return@forEach + val title = history.video.title.sanitizedReconstructionText() + json.writeString("${history.video.url}|||${history.watchedAt}|||${history.positionSeconds}|||$title") + } + } + + private fun writeWatchLater(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "watch_later") { json -> + source.forEach(PortabilityCategory.WATCH_LATER) { record -> + (record as? PortabilityWatchLater)?.let { json.writeString(it.video.url) } + } + } + + private fun writePlaylists(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "playlists") { json -> + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + val playlist = record as? PortabilityPlaylist ?: return@forEach + val value = buildString { + append(playlist.name.sanitizedReconstructionText()) + append(":::") + append(playlist.sourceId.sanitizedReconstructionText()) + source.forEachChild(PortabilityCategory.PLAYLISTS, playlist.stableKey().removePrefix("playlist:")) { child -> + val item = child as? PortabilityPlaylistVideo ?: return@forEachChild + append('\n').append(item.video.url.replace("\n", "")) + } + } + json.writeString(value) + } + } + + private fun writeGroups(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "subscription_groups") { json -> + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + val group = record as? PortabilitySubscriptionGroup ?: return@forEach + val value = buildGrayjayGroup(source, group) + require(value.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "Grayjay group is too large" } + json.writeString(value) + } + } + + private fun buildGrayjayGroup(source: PortabilityRecordSource, group: PortabilitySubscriptionGroup): String { + val bytes = java.io.ByteArrayOutputStream() + PortabilityJsonFactory.createGenerator(bytes).use { json -> + json.writeStartObject() + json.writeStringField("id", UUID.nameUUIDFromBytes(group.stableKey().toByteArray()).toString()) + json.writeStringField("name", group.name) + json.writeArrayFieldStart("urls") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, group.name) { child -> + (child as? PortabilitySubscriptionGroupMembership)?.let { json.writeString(it.channelUrl) } + } + json.writeEndArray() + json.writeNumberField("priority", 99) + json.writeEndObject() + } + return bytes.toString(Charsets.UTF_8) + } + + private fun writeObjectEntry(zip: ZipOutputStream, name: String, values: Map<*, *>) { + zip.putNextEntry(ZipEntry(name)) + val json = generator(zip) + json.writeStartObject() + values.forEach { (key, value) -> json.writeStringField(key.toString(), value.toString()) } + json.writeEndObject() + json.close() + zip.closeEntry() + } + + private inline fun writeStore(zip: ZipOutputStream, name: String, block: (JsonGenerator) -> Unit) { + zip.putNextEntry(ZipEntry("stores/$name")) + val json = generator(zip) + json.writeStartArray() + block(json) + json.writeEndArray() + json.close() + zip.closeEntry() + } + + private fun generator(output: OutputStream): JsonGenerator = PortabilityJsonFactory.createGenerator(output).apply { + disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) + } +} + +private fun String.sanitizedReconstructionText(): String = replace("|||", " ").replace(":::", " ").replace("\n", " ") diff --git a/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt new file mode 100644 index 00000000..98a3e439 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt @@ -0,0 +1,81 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream +import kotlin.io.path.outputStream + +class GrayjayPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports current grayjay reconstruction stores`() { + val archive = directory.resolve("grayjay.zip") + writeGrayjayFixture(archive) + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = GrayjayPortabilityAdapter() + + assertEquals("1", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + spool.delete() + } + + @Test + fun `exports a grayjay archive that round trips through the adapter`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://www.youtube.com/channel/UC1", "Channel")) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://www.youtube.com/channel/UC1")) + source.write(PortabilityHistory(PortabilityVideo("https://youtu.be/v1", "Title"), 100, 25)) + source.write(PortabilityPlaylist("p1", "Playlist")) + source.write(PortabilityPlaylistVideo("p1", 0, PortabilityVideo("https://youtu.be/v1"))) + source.write(PortabilityWatchLater(PortabilityVideo("https://youtu.be/v2"))) + val archive = directory.resolve("export.zip") + archive.outputStream().use { GrayjayPortabilityAdapter().encode(source, it, source.categories()) } + + ZipFile(archive.toFile()).use { zip -> + assertTrue(zip.getInputStream(zip.getEntry("exportInfo")).reader().readText().contains("\"version\":\"1\"")) + assertTrue(zip.getInputStream(zip.getEntry("stores/playlists")).reader().readText().contains("Playlist:::p1")) + } + val restored = PortabilitySpool.create(directory) + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + GrayjayPortabilityAdapter().decode(input, restored) + assertEquals(2L, restored.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(2L, restored.counts()[PortabilityCategory.PLAYLISTS]) + source.delete() + restored.delete() + } + + private fun writeGrayjayFixture(path: Path) { + ZipOutputStream(path.outputStream()).use { zip -> + entry(zip, "exportInfo", "{\"version\":\"1\"}") + entry(zip, "stores/subscriptions", "[\"https://www.youtube.com/channel/UC1\"]") + entry(zip, "stores/subscription_groups", "[\"{\\\"name\\\":\\\"News\\\",\\\"urls\\\":[\\\"https://www.youtube.com/channel/UC1\\\"]}\"]") + entry(zip, "stores/history", "[\"https://youtu.be/v1|||100|||25|||Title\"]") + entry(zip, "stores/playlists", "[\"Playlist:::p1\\nhttps://youtu.be/v1\\nhttps://youtu.be/v2\"]") + entry(zip, "stores/watch_later", "[\"https://youtu.be/v2\"]") + entry(zip, "plugins", "{}") + entry(zip, "plugin_settings", "{}") + } + } + + private fun entry(zip: ZipOutputStream, name: String, value: String) { + zip.putNextEntry(ZipEntry(name)) + zip.write(value.toByteArray()) + zip.closeEntry() + } +} From 22d0e37c8f89a2bed59f1fbf344941532107c5e3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:22 +0200 Subject: [PATCH 30/68] feat: add ViewTube import support --- .../portability/ViewTubePortabilityAdapter.kt | 56 ++++++ .../portability/ViewTubePortabilityReader.kt | 160 ++++++++++++++++++ .../ViewTubePortabilityAdapterTest.kt | 66 ++++++++ 3 files changed, 282 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt new file mode 100644 index 00000000..efb780da --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class ViewTubePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.VIEW_TUBE, + 1, + setOf( + viewTubeCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + viewTubeCapability(PortabilityCategory.HISTORY, PortabilityFidelity.COMPLETE), + viewTubeCapability(PortabilityCategory.PROGRESS, PortabilityFidelity.COMPLETE), + ), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + if ("user.json" !in archive.names) return null + ZipFile(input.path.toFile()).use { zip -> + val entry = zip.getEntry("user.json") ?: return null + zip.getInputStream(entry).buffered().use { stream -> + val probe = stream.readNBytes(PortabilityLimits.PROBE_BYTES).decodeToString() + if (!probe.contains("\"username\"") || !probe.contains("\"subscriptions\"")) return null + if (!probe.contains("\"history\"") || !probe.contains("\"settings\"")) return null + } + } + return PortabilityDetection(PortabilityFormat.VIEW_TUBE, null, 99, "ViewTube user.json export") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported ViewTube backup" } + ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry("user.json")) + zip.getInputStream(entry).buffered().use { inputStream -> + PortabilityJsonFactory.createParser(inputStream).use { parser -> + ViewTubePortabilityReader.read(parser, sink) + } + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ): Unit = error("ViewTube does not provide a compatible full-backup import") +} + +private fun viewTubeCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt new file mode 100644 index 00000000..337594cc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt @@ -0,0 +1,160 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import java.time.Instant + +internal object ViewTubePortabilityReader { + fun read(parser: JsonParser, sink: PortabilityRecordSink) { + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "subscriptions" -> readContainer(parser, token, "channels") { readSubscription(parser, sink) } + "history" -> readContainer(parser, token, "videos") { readHistory(parser, sink) } + "settings" -> { + parser.skipChildren() + sink.issue(PortabilityIssue(PortabilityCategory.SETTINGS, "viewtube_settings_ignored", "ViewTube settings are not portable to TypeType")) + } + else -> parser.skipChildren() + } + } + } + + private fun readContainer(parser: JsonParser, token: JsonToken, arrayField: String, item: () -> Unit) { + if (token != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val value = parser.nextToken() + if (field == arrayField && value == JsonToken.START_ARRAY) { + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "$arrayField contains too many records" } + item() + } + } else { + parser.skipChildren() + } + } + } + + private fun readSubscription(parser: JsonParser, sink: PortabilityRecordSink) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + var id = "" + var name = "" + var url = "" + var avatar = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "authorId" -> id = parser.textOrEmpty() + "author" -> name = parser.textOrEmpty() + "authorUrl" -> url = parser.textOrEmpty() + "authorThumbnailUrl" -> avatar = parser.textOrEmpty() + "authorThumbnails" -> avatar = avatar.ifBlank { readFirstUrl(parser) } + else -> parser.skipChildren() + } + } + val channelUrl = url.ifBlank { youtubeChannelUrl(id) } + if (channelUrl.isNotBlank()) sink.write(PortabilitySubscription(channelUrl, name, avatar)) + } + + private fun readHistory(parser: JsonParser, sink: PortabilityRecordSink) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + val fields = ViewTubeHistoryFields() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "videoId" -> fields.videoId = parser.textOrEmpty() + "progressSeconds" -> fields.progress = parser.longOrZero() + "lengthSeconds" -> fields.duration = parser.longOrZero() + "lastVisit" -> fields.watchedAt = parseInstant(parser.textOrEmpty()) + "videoDetails" -> readVideoDetails(parser, fields) + else -> parser.skipChildren() + } + } + val url = youtubeVideoUrl(fields.videoId) + if (url.isBlank()) return + val video = PortabilityVideo( + url = url, + title = fields.title, + thumbnailUrl = fields.thumbnail, + durationSeconds = fields.duration, + channelName = fields.channelName, + channelUrl = youtubeChannelUrl(fields.channelId), + channelAvatarUrl = fields.channelAvatar, + viewCount = fields.views, + publishedAt = fields.publishedAt, + ) + sink.write(PortabilityHistory(video, fields.watchedAt, fields.progress)) + sink.write(PortabilityProgress(url, fields.progress, fields.watchedAt)) + } + + private fun readVideoDetails(parser: JsonParser, fields: ViewTubeHistoryFields) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "videoId" -> fields.videoId = parser.textOrEmpty() + "title" -> fields.title = parser.textOrEmpty() + "author" -> fields.channelName = parser.textOrEmpty() + "authorId" -> fields.channelId = parser.textOrEmpty() + "authorThumbnailUrl" -> fields.channelAvatar = parser.textOrEmpty() + "lengthSeconds" -> fields.duration = parser.longOrZero() + "viewCount" -> fields.views = parser.longOrZero() + "published" -> fields.publishedAt = parser.longOrZero() + "videoThumbnails" -> fields.thumbnail = readFirstUrl(parser) + else -> parser.skipChildren() + } + } + } + + private fun readFirstUrl(parser: JsonParser): String { + if (parser.currentToken() != JsonToken.START_ARRAY) return parser.skipChildren().let { "" } + var url = "" + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + if (field == "url" && url.isBlank()) url = parser.textOrEmpty() else parser.skipChildren() + } + } + return url + } + + private fun parseInstant(value: String): Long = runCatching { Instant.parse(value).epochSecond }.getOrDefault(0L) +} + +private data class ViewTubeHistoryFields( + var videoId: String = "", + var progress: Long = 0, + var duration: Long = 0, + var watchedAt: Long = 0, + var title: String = "", + var thumbnail: String = "", + var channelName: String = "", + var channelId: String = "", + var channelAvatar: String = "", + var views: Long = 0, + var publishedAt: Long = -1, +) diff --git a/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt new file mode 100644 index 00000000..601fb0eb --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.outputStream + +class ViewTubePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports viewtube subscriptions history and progress`() { + val archive = directory.resolve("viewtube.zip") + ZipOutputStream(archive.outputStream()).use { zip -> + zip.putNextEntry(ZipEntry("user.json")) + zip.write(VIEW_TUBE_USER.toByteArray()) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = ViewTubePortabilityAdapter() + + assertEquals(PortabilityFormat.VIEW_TUBE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + var history: PortabilityHistory? = null + spool.forEach(PortabilityCategory.HISTORY) { history = it as PortabilityHistory } + assertEquals("https://www.youtube.com/watch?v=video1", history?.video?.url) + assertEquals("A video", history?.video?.title) + assertEquals(42L, history?.positionSeconds) + assertTrue(spool.issues().any { it.code == "viewtube_settings_ignored" }) + spool.delete() + } + + @Test + fun `does not advertise a viewtube export that viewtube cannot restore`() { + val capability = ViewTubePortabilityAdapter().descriptor.capabilities + assertTrue(capability.isNotEmpty()) + assertTrue(capability.all { it.directions == setOf(PortabilityDirection.IMPORT) }) + assertNotNull(capability.firstOrNull { it.category == PortabilityCategory.HISTORY }) + } +} + +private val VIEW_TUBE_USER = """ + { + "username":"alice", + "subscriptions":{"channels":[{ + "authorId":"UC1","author":"Channel","authorUrl":"https://www.youtube.com/channel/UC1", + "authorThumbnails":[{"url":"avatar","width":88,"height":88}] + }],"channelCount":1}, + "history":{"videos":[{ + "videoId":"video1","progressSeconds":42,"lengthSeconds":120,"lastVisit":"2026-08-20T10:15:30Z", + "videoDetails":{"videoId":"video1","title":"A video","author":"Channel","authorId":"UC1", + "videoThumbnails":[{"url":"thumb","width":480,"height":360}],"viewCount":12,"lengthSeconds":120} + }],"videoCount":1}, + "settings":{"theme":"dark"} + } +""".trimIndent() From feaff4aec8df6b693baf22b85f67713951357690 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:23 +0200 Subject: [PATCH 31/68] feat: add Materialious portability support --- .../MaterialiousPortabilityAdapter.kt | 75 +++++++++++++++++++ .../MaterialiousPortabilityAdapterTest.kt | 49 ++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt new file mode 100644 index 00000000..44bc7ee4 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt @@ -0,0 +1,75 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class MaterialiousPortabilityAdapter : PortabilityAdapter { + private val opml = OpmlPortabilityAdapter(PortabilityFormat.MATERIALIOUS, autoDetect = false) + + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.MATERIALIOUS, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.PARTIAL, + ), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + if (opml.detect(input) != null) { + return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "opml", 85, "Materialious-compatible OPML") + } + val probe = input.probe.decodeToString() + if (!probe.trimStart().startsWith("{") || !probe.contains("\"subscriptions\"")) return null + if (probe.contains("\"app_version")) return null + val confidence = if (input.filename.contains("materialious", ignoreCase = true)) 99 else 86 + return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "invidious-subscriptions", confidence, "Materialious Invidious subscription JSON") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + val detection = requireNotNull(detect(input)) { "Unsupported Materialious export" } + if (detection.formatVersion == "opml") return opml.decode(input, sink) + input.withJsonParser { parser -> + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + if (field == "subscriptions" && token == JsonToken.START_ARRAY) readSubscriptions(parser, sink) + else parser.skipChildren() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + require(PortabilityCategory.SUBSCRIPTIONS in categories) { "Materialious export requires subscriptions" } + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + (record as? PortabilitySubscription)?.let { json.writeString(youtubeId(it.channelUrl)) } + } + json.writeEndArray() + json.writeEndObject() + } + } + + private fun readSubscriptions(parser: com.fasterxml.jackson.core.JsonParser, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "Materialious export contains too many subscriptions" } + val channelUrl = youtubeChannelUrl(parser.textOrEmpty()) + if (channelUrl.isNotBlank()) sink.write(PortabilitySubscription(channelUrl)) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt new file mode 100644 index 00000000..07d0183a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class MaterialiousPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports and exports materialious invidious subscription json`() { + val file = directory.resolve("materialious-export.json") + Files.writeString(file, """{"subscriptions":["UC1","UC2"]}""") + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val adapter = MaterialiousPortabilityAdapter() + val spool = PortabilitySpool.create(directory) + + assertEquals("invidious-subscriptions", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("\"subscriptions\":[\"UC1\",\"UC2\"]")) + spool.delete() + } + + @Test + fun `accepts the opml format supported by materialious`() { + val file = directory.resolve("materialious.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + val adapter = MaterialiousPortabilityAdapter() + + assertEquals("opml", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + spool.delete() + } +} From 6f9a0285a4bc1e63b1627a7e046768a572563e55 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:24 +0200 Subject: [PATCH 32/68] feat: add OPML portability adapters --- .../portability/OpmlPortabilityAdapter.kt | 116 ++++++++++++++++++ .../portability/OpmlPortabilityAdapterTest.kt | 65 ++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt new file mode 100644 index 00000000..5b1438b5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt @@ -0,0 +1,116 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import javax.xml.stream.XMLInputFactory +import javax.xml.stream.XMLOutputFactory +import javax.xml.stream.XMLStreamConstants + +class OpmlPortabilityAdapter( + private val format: PortabilityFormat = PortabilityFormat.OPML, + override val autoDetect: Boolean = format == PortabilityFormat.OPML, +) : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.PARTIAL, + ), + ), + "opml", + "application/xml", + ) + + init { + require(format in COMPATIBLE_FORMATS) { "Format does not use OPML" } + } + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString().trimStart() + if (!probe.startsWith("<") || !probe.contains("]*version=[\"']([^\"']+)", RegexOption.IGNORE_CASE) + .find(probe)?.groupValues?.get(1) + return PortabilityDetection(format, version, 92, "OPML document root") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + val factory = secureInputFactory() + java.nio.file.Files.newInputStream(input.path).buffered().use { stream -> + val reader = factory.createXMLStreamReader(stream) + var outlines = 0 + try { + while (reader.hasNext()) { + if (reader.next() != XMLStreamConstants.START_ELEMENT || reader.localName != "outline") continue + require(outlines++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "OPML contains too many outlines" } + val rawUrl = reader.attribute("xmlUrl").ifBlank { reader.attribute("url") } + val channelUrl = normalizeOpmlChannel(rawUrl) + if (channelUrl.isNotBlank()) { + val name = reader.attribute("title").ifBlank { reader.attribute("text") } + sink.write(PortabilitySubscription(channelUrl, name)) + } + } + } finally { + reader.close() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + require(PortabilityCategory.SUBSCRIPTIONS in categories) { "OPML export requires subscriptions" } + val writer = XMLOutputFactory.newFactory().createXMLStreamWriter(output, Charsets.UTF_8.name()) + writer.writeStartDocument(Charsets.UTF_8.name(), "1.0") + writer.writeStartElement("opml") + writer.writeAttribute("version", "2.0") + writer.writeStartElement("head") + writer.writeStartElement("title") + writer.writeCharacters("TypeType subscriptions") + writer.writeEndElement() + writer.writeEndElement() + writer.writeStartElement("body") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + val channelId = youtubeId(item.channelUrl) + writer.writeEmptyElement("outline") + writer.writeAttribute("text", item.name.ifBlank { channelId }) + writer.writeAttribute("title", item.name.ifBlank { channelId }) + writer.writeAttribute("type", "rss") + writer.writeAttribute("xmlUrl", "https://www.youtube.com/feeds/videos.xml?channel_id=$channelId") + writer.writeAttribute("htmlUrl", item.channelUrl) + } + writer.writeEndElement() + writer.writeEndElement() + writer.writeEndDocument() + writer.close() + } + + private companion object { + val COMPATIBLE_FORMATS = setOf( + PortabilityFormat.OPML, + PortabilityFormat.MATERIALIOUS, + PortabilityFormat.SKY_TUBE, + PortabilityFormat.YOUTUBE_LOCAL, + ) + } +} + +private fun secureInputFactory(): XMLInputFactory = XMLInputFactory.newFactory().apply { + setProperty(XMLInputFactory.SUPPORT_DTD, false) + setProperty("javax.xml.stream.isSupportingExternalEntities", false) +} + +private fun javax.xml.stream.XMLStreamReader.attribute(name: String): String = + (0 until attributeCount).firstOrNull { getAttributeLocalName(it) == name }?.let(::getAttributeValue).orEmpty() + +private fun normalizeOpmlChannel(value: String): String { + if (value.isBlank()) return "" + val channelId = Regex("[?&]channel_id=([^&#]+)").find(value)?.groupValues?.get(1) + return if (channelId != null) youtubeChannelUrl(channelId) else youtubeChannelUrl(value) +} diff --git a/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt new file mode 100644 index 00000000..2f8d4aa2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt @@ -0,0 +1,65 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class OpmlPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter imports feeds and writes portable opml`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + val adapter = OpmlPortabilityAdapter() + + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("channel_id=UC1")) + spool.delete() + } + + @Test + fun `adapter does not resolve external entities`() { + val file = directory.resolve("unsafe.opml") + Files.writeString(file, """]>""") + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + + assertThrows(Exception::class.java) { OpmlPortabilityAdapter().decode(input, spool) } + spool.delete() + } + + @Test + fun `skytube and youtube local use their supported opml exchange`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + listOf(PortabilityFormat.SKY_TUBE, PortabilityFormat.YOUTUBE_LOCAL).forEach { format -> + val adapter = OpmlPortabilityAdapter(format, autoDetect = false) + val spool = PortabilitySpool.create(directory) + assertEquals(format, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("channel_id=UC1")) + spool.delete() + } + } +} From a1fce7fe701f668eff611e3a499aff6db015b619 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:39 +0200 Subject: [PATCH 33/68] feat: model NewPipe archive databases --- .../portability/NewPipeArchiveCapabilities.kt | 46 ++++++++++++++ .../portability/NewPipeArchiveDatabase.kt | 63 +++++++++++++++++++ .../portability/NewPipeArchiveSchema.kt | 62 ++++++++++++++++++ .../portability/NewPipeArchiveTarget.kt | 10 +++ .../server/portability/NewPipeProvider.kt | 20 ++++++ 5 files changed, 201 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt new file mode 100644 index 00000000..3c99df75 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +internal fun newPipeArchiveCapabilities(): Set = + archiveImportCapabilities().mapTo(linkedSetOf()) { capability -> + PortabilityCapability( + capability.category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + when (capability.category) { + PortabilityCategory.PLAYLISTS -> PortabilityFidelity.PARTIAL + else -> PortabilityFidelity.COMPLETE + }, + ) + } + +internal fun newPipeArchiveExportIssues( + source: PortabilityRecordSource, + categories: Set, + target: NewPipeArchiveTarget, +): List { + val counts = linkedMapOf() + categories.forEach { category -> + source.forEach(category) { record -> + record.urls().forEach { url -> + if (!NewPipeProvider.supported(url, target)) counts[category] = (counts[category] ?: 0L) + 1L + } + } + } + return counts.map { (category, count) -> + PortabilityIssue( + category, + "unsupported_provider", + "${target.name.replace('_', ' ')} cannot represent one or more provider records", + count, + ) + } +} + +private fun PortabilityRecord.urls(): List = when (this) { + is PortabilitySubscription -> listOf(channelUrl) + is PortabilitySubscriptionGroupMembership -> listOf(channelUrl) + is PortabilityHistory -> listOf(video.url) + is PortabilityPlaylistVideo -> listOf(video.url) + is PortabilityProgress -> listOf(videoUrl) + is PortabilitySavedPlaylist -> listOf(url) + else -> emptyList() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt new file mode 100644 index 00000000..ad822179 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt @@ -0,0 +1,63 @@ +package dev.typetype.server.portability + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.file.Files +import java.sql.Connection +import java.sql.DriverManager +import java.util.zip.ZipFile + +internal object NewPipeArchiveDatabase { + private val databaseNames = setOf("newpipe.db", "pipepipe.db") + + fun userVersion(input: PortabilityInput): Int? { + val entryName = databaseEntry(input) ?: return null + return ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry(entryName)) + val header = zip.getInputStream(entry).use { it.readNBytes(SQLITE_HEADER_BYTES) } + if (header.size < SQLITE_HEADER_BYTES || !header.startsWith(SQLITE_SIGNATURE)) return null + ByteBuffer.wrap(header, USER_VERSION_OFFSET, Int.SIZE_BYTES) + .order(ByteOrder.BIG_ENDIAN) + .int + } + } + + fun read(input: PortabilityInput, block: (Connection) -> T): T { + val entryName = requireNotNull(databaseEntry(input)) { "Backup database is missing" } + val extracted = Files.createTempFile(requireNotNull(input.path.parent), "portability-newpipe-", ".sqlite") + try { + ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry(entryName)) + zip.getInputStream(entry).use { source -> + Files.newOutputStream(extracted).use { target -> source.copyTo(target) } + } + } + require(Files.size(extracted) in 1..PortabilityLimits.MAX_ARCHIVE_ENTRY_BYTES) { + "Backup database is outside the allowed range" + } + Class.forName("org.sqlite.JDBC") + return DriverManager.getConnection("jdbc:sqlite:file:${extracted.toAbsolutePath()}?mode=ro&immutable=1").use { connection -> + connection.createStatement().use { statement -> + statement.execute("PRAGMA query_only = ON") + statement.execute("PRAGMA trusted_schema = OFF") + statement.execute("PRAGMA cell_size_check = ON") + } + block(connection) + } + } finally { + Files.deleteIfExists(extracted) + } + } + + private fun databaseEntry(input: PortabilityInput): String? = input.archive?.entries + ?.map(PortabilityArchiveEntry::name) + ?.filter { it.substringAfterLast('/').lowercase() in databaseNames } + ?.singleOrNull() + + private fun ByteArray.startsWith(prefix: ByteArray): Boolean = + size >= prefix.size && prefix.indices.all { this[it] == prefix[it] } + + private const val SQLITE_HEADER_BYTES = 64 + private const val USER_VERSION_OFFSET = 60 + private val SQLITE_SIGNATURE = "SQLite format 3\u0000".toByteArray() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt new file mode 100644 index 00000000..e83b543f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import java.sql.Connection + +internal object NewPipeArchiveSchema { + fun create(db: Connection, target: NewPipeArchiveTarget) { + db.createStatement().use { statement -> + statement.execute("PRAGMA foreign_keys = ON") + statement.execute("PRAGMA journal_mode = DELETE") + statements(target).forEach(statement::execute) + statement.execute("PRAGMA user_version = ${target.databaseVersion}") + statement.execute("CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)") + statement.execute("INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42,'${target.identityHash}')") + } + db.commit() + } + + private fun statements(target: NewPipeArchiveTarget): List = listOf( + "CREATE TABLE subscriptions (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT, name TEXT, avatar_url TEXT, subscriber_count INTEGER, description TEXT, notification_mode INTEGER NOT NULL)", + "CREATE UNIQUE INDEX index_subscriptions_service_id_url ON subscriptions(service_id,url)", + "CREATE TABLE search_history (creation_date INTEGER, service_id INTEGER NOT NULL, search TEXT, id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL)", + "CREATE INDEX index_search_history_search ON search_history(search)", + streams(target), + "CREATE UNIQUE INDEX index_streams_service_id_url ON streams(service_id,url)", + "CREATE TABLE stream_history (stream_id INTEGER NOT NULL, access_date INTEGER NOT NULL, repeat_count INTEGER NOT NULL, PRIMARY KEY(stream_id,access_date), FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE)", + "CREATE INDEX index_stream_history_stream_id ON stream_history(stream_id)", + "CREATE TABLE stream_state (stream_id INTEGER NOT NULL PRIMARY KEY, progress_time INTEGER NOT NULL, FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE)", + playlists(target), + "CREATE TABLE playlist_stream_join (playlist_id INTEGER NOT NULL, stream_id INTEGER NOT NULL, join_index INTEGER NOT NULL, PRIMARY KEY(playlist_id,join_index), FOREIGN KEY(playlist_id) REFERENCES playlists(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE UNIQUE INDEX index_playlist_stream_join_playlist_id_join_index ON playlist_stream_join(playlist_id,join_index)", + "CREATE INDEX index_playlist_stream_join_stream_id ON playlist_stream_join(stream_id)", + "CREATE TABLE remote_playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, name TEXT, url TEXT, thumbnail_url TEXT, uploader TEXT, display_index INTEGER NOT NULL, stream_count INTEGER)", + "CREATE UNIQUE INDEX index_remote_playlists_service_id_url ON remote_playlists(service_id,url)", + "CREATE TABLE feed (stream_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(stream_id,subscription_id), FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE INDEX index_feed_subscription_id ON feed(subscription_id)", + "CREATE TABLE feed_group (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, icon_id INTEGER NOT NULL, sort_order INTEGER NOT NULL)", + "CREATE INDEX index_feed_group_sort_order ON feed_group(sort_order)", + "CREATE TABLE feed_group_subscription_join (group_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(group_id,subscription_id), FOREIGN KEY(group_id) REFERENCES feed_group(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE INDEX index_feed_group_subscription_join_subscription_id ON feed_group_subscription_join(subscription_id)", + "CREATE TABLE feed_last_updated (subscription_id INTEGER NOT NULL PRIMARY KEY, last_updated INTEGER, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + ) + targetIndexes(target) + + private fun streams(target: NewPipeArchiveTarget): String { + val paid = if (target.pipePipe) ", is_paid INTEGER NOT NULL" else "" + return "CREATE TABLE streams (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, stream_type TEXT NOT NULL, duration INTEGER NOT NULL, uploader TEXT NOT NULL, uploader_url TEXT, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER, is_upload_date_approximation INTEGER$paid)" + } + + private fun playlists(target: NewPipeArchiveTarget): String = if (target.pipePipe) { + "CREATE TABLE playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT, thumbnail_url TEXT, display_index INTEGER NOT NULL)" + } else { + "CREATE TABLE playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT, is_thumbnail_permanent INTEGER NOT NULL, thumbnail_stream_id INTEGER NOT NULL, display_index INTEGER NOT NULL)" + } + + private fun targetIndexes(target: NewPipeArchiveTarget): List = if (target.pipePipe) { + listOf( + "CREATE INDEX index_playlists_name ON playlists(name)", + "CREATE INDEX index_remote_playlists_name ON remote_playlists(name)", + ) + } else { + emptyList() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt new file mode 100644 index 00000000..61bf8305 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.portability + +internal enum class NewPipeArchiveTarget( + val databaseVersion: Int, + val identityHash: String, + val pipePipe: Boolean, +) { + NEW_PIPE(9, "7591e8039faa74d8c0517dc867af9d3e", false), + PIPE_PIPE(901, "d505dd6c0be6a80da07aa980bd361064", true), +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt new file mode 100644 index 00000000..084cf6bc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.portability + +import java.net.URI + +internal object NewPipeProvider { + fun serviceId(url: String): Int? { + val host = runCatching { URI(url.trim()).host?.lowercase() }.getOrNull() ?: return null + return when { + host == "youtube.com" || host == "youtu.be" || host.endsWith(".youtube.com") -> 0 + host == "bilibili.com" || host.endsWith(".bilibili.com") || host == "b23.tv" -> 5 + host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") || host == "nico.ms" -> 6 + else -> null + } + } + + fun supported(url: String, target: NewPipeArchiveTarget): Boolean { + val id = serviceId(url) ?: return false + return target.pipePipe || id == 0 + } +} From 171872c734eb8f329291fc2fb2d2c00d4348dbcf Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:39 +0200 Subject: [PATCH 34/68] feat: decode NewPipe account backups --- .../NewPipeDatabasePortabilityReader.kt | 151 ++++++++++++++++++ .../portability/NewPipePortabilityAdapter.kt | 104 ++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt new file mode 100644 index 00000000..87d057f8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt @@ -0,0 +1,151 @@ +package dev.typetype.server.portability + +import java.sql.Connection +import java.sql.ResultSet + +internal object NewPipeDatabasePortabilityReader { + fun read(connection: Connection, sink: PortabilityRecordSink) { + subscriptions(connection, sink) + groups(connection, sink) + history(connection, sink) + playlists(connection, sink) + progress(connection, sink) + searchHistory(connection, sink) + } + + private fun subscriptions(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.SUBSCRIPTIONS, + "subscriptions", + "SELECT service_id, url, name, avatar_url FROM subscriptions ORDER BY uid", + ) { row -> + if (row.int("service_id") == 0) { + sink.write(PortabilitySubscription(row.string("url"), row.string("name"), row.string("avatar_url"))) + } else { + sink.issue(PortabilityIssue(PortabilityCategory.SUBSCRIPTIONS, "unsupported_subscription_provider", "A non-YouTube subscription was skipped")) + } + } + + private fun groups(db: Connection, sink: PortabilityRecordSink) { + sink.readTable(db, PortabilityCategory.SUBSCRIPTION_GROUPS, "feed_group", "SELECT uid, name FROM feed_group ORDER BY sort_order, uid") { row -> + sink.write(PortabilitySubscriptionGroup(row.string("name"))) + } + if (!db.hasTables("feed_group_subscription_join", "feed_group", "subscriptions")) return + db.each( + """ + SELECT g.name AS group_name, s.url AS channel_url + FROM feed_group_subscription_join j + JOIN feed_group g ON g.uid = j.group_id + JOIN subscriptions s ON s.uid = j.subscription_id + ORDER BY j.group_id, j.subscription_id + """.trimIndent(), + ) { row -> + sink.write(PortabilitySubscriptionGroupMembership(row.string("group_name"), row.string("channel_url"))) + } + } + + private fun history(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.HISTORY, + "stream_history", + """ + SELECT h.access_date, s.url, s.title, s.duration, s.uploader, s.uploader_url, s.thumbnail_url + FROM stream_history h JOIN streams s ON s.uid = h.stream_id + ORDER BY h.access_date, h.stream_id + """.trimIndent(), + requiredTables = arrayOf("streams"), + ) { row -> + sink.write( + PortabilityHistory( + row.video(), + row.long("access_date"), + ), + ) + } + + private fun playlists(db: Connection, sink: PortabilityRecordSink) { + sink.readTable(db, PortabilityCategory.PLAYLISTS, "playlists", "SELECT uid, name FROM playlists ORDER BY display_index, uid") { row -> + sink.write(PortabilityPlaylist(row.long("uid").toString(), row.string("name"))) + } + if (db.hasTables("playlist_stream_join", "streams")) { + db.each( + """ + SELECT j.playlist_id, j.join_index, s.url, s.title, s.duration, s.uploader, s.uploader_url, s.thumbnail_url + FROM playlist_stream_join j JOIN streams s ON s.uid = j.stream_id + ORDER BY j.playlist_id, j.join_index + """.trimIndent(), + ) { row -> + sink.write(PortabilityPlaylistVideo(row.long("playlist_id").toString(), row.int("join_index"), row.video())) + } + } + sink.readTable( + db, + PortabilityCategory.SAVED_PLAYLISTS, + "remote_playlists", + "SELECT uid, url, name, thumbnail_url, uploader, stream_count FROM remote_playlists ORDER BY display_index, uid", + ) { row -> + sink.write( + PortabilitySavedPlaylist( + row.long("uid").toString(), row.string("url"), row.string("name"), + row.string("thumbnail_url"), row.string("uploader"), row.long("stream_count"), + ), + ) + } + } + + private fun progress(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.PROGRESS, + "stream_state", + "SELECT s.url, state.progress_time FROM stream_state state JOIN streams s ON s.uid = state.stream_id", + requiredTables = arrayOf("streams"), + ) { row -> + sink.write(PortabilityProgress(row.string("url"), row.long("progress_time"))) + } + + private fun searchHistory(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.SEARCH_HISTORY, + "search_history", + "SELECT search, creation_date FROM search_history ORDER BY creation_date, id", + ) { row -> + sink.write(PortabilitySearchHistory(row.string("search"), row.long("creation_date"))) + } +} + +private fun PortabilityRecordSink.readTable( + db: Connection, + category: PortabilityCategory, + table: String, + sql: String, + requiredTables: Array = emptyArray(), + block: (SqliteRow) -> Unit, +) { + if (!db.hasTables(table, *requiredTables)) return + markCategory(category) + db.each(sql, block) +} + +private fun Connection.each(sql: String, block: (SqliteRow) -> Unit) { + prepareStatement(sql).use { statement -> + statement.fetchSize = 256 + statement.executeQuery().use { rows -> while (rows.next()) block(SqliteRow(rows)) } + } +} + +private fun Connection.hasTables(vararg names: String): Boolean = names.all { name -> + prepareStatement("SELECT 1 FROM sqlite_master WHERE type='table' AND lower(name)=lower(?)").use { statement -> + statement.setString(1, name) + statement.executeQuery().use(ResultSet::next) + } +} + +private class SqliteRow(private val row: ResultSet) { + fun string(name: String): String = row.getString(name) ?: "" + fun int(name: String): Int = row.getInt(name) + fun long(name: String): Long = row.getLong(name) + fun video() = PortabilityVideo( + string("url"), string("title"), string("thumbnail_url"), long("duration"), + string("uploader"), string("uploader_url"), + ) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt new file mode 100644 index 00000000..12f228aa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class NewPipePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.NEW_PIPE, + adapterVersion = 3, + capabilities = newPipeArchiveCapabilities(), + defaultExtension = "zip", + contentType = "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) { + val version = NewPipeArchiveDatabase.userVersion(input) ?: return null + if (version !in 1 until PIPE_PIPE_DATABASE_VERSION) return null + return PortabilityDetection(PortabilityFormat.NEW_PIPE, version.toString(), 100, "NewPipe SQLite schema version") + } + val probe = input.probe.decodeToString() + if (!probe.contains("\"subscriptions\"") || !probe.contains("\"app_version")) return null + return PortabilityDetection(PortabilityFormat.NEW_PIPE, null, 98, "NewPipe subscription fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + if (input.archive != null) { + requireNotNull(detect(input)) { "Unsupported NewPipe backup" } + NewPipeArchiveDatabase.read(input) { NewPipeDatabasePortabilityReader.read(it, sink) } + return + } + decodeSubscriptions(input, sink) + } + + private fun decodeSubscriptions(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var foundSubscriptions = false + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val valueToken = parser.nextToken() + if (field == "subscriptions" && valueToken == JsonToken.START_ARRAY) { + foundSubscriptions = true + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readSubscriptions(parser, sink) + } else { + parser.skipChildren() + } + } + require(foundSubscriptions) { "NewPipe backup does not contain subscriptions" } + } + + private companion object { + const val PIPE_PIPE_DATABASE_VERSION = 900 + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List { + return super.assessExport(source, categories) + + newPipeArchiveExportIssues(source, categories, NewPipeArchiveTarget.NEW_PIPE) + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + NewPipeArchivePortabilityWriter.write(source, output, categories, NewPipeArchiveTarget.NEW_PIPE) + } + + private fun readSubscriptions( + parser: com.fasterxml.jackson.core.JsonParser, + sink: PortabilityRecordSink, + ) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(parser.currentToken() == JsonToken.START_OBJECT) { "Invalid NewPipe subscription" } + var serviceId = -1 + var url = "" + var name = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "service_id" -> serviceId = parser.intValue + "url" -> url = parser.textOrEmpty() + "name" -> name = parser.textOrEmpty() + else -> parser.skipChildren() + } + } + if (serviceId != 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.SUBSCRIPTIONS, + "unsupported_subscription_provider", + "A non-YouTube NewPipe subscription was skipped", + ), + ) + } else if (url.isNotBlank()) { + sink.write(PortabilitySubscription(url.trim(), name.trim())) + } + } + } +} From 49156e3139c35c74f20c130bda33655f22f55e49 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:39 +0200 Subject: [PATCH 35/68] feat: encode NewPipe archive records --- .../NewPipeArchivePortabilityWriter.kt | 34 ++++ .../portability/NewPipeArchiveRecordWriter.kt | 179 ++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt new file mode 100644 index 00000000..96d2b206 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.nio.file.Files +import java.sql.DriverManager +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +internal object NewPipeArchivePortabilityWriter { + fun write( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + target: NewPipeArchiveTarget, + ) { + val database = Files.createTempFile("typetype-portability-", ".db") + try { + DriverManager.getConnection("jdbc:sqlite:$database").use { db -> + db.autoCommit = false + NewPipeArchiveSchema.create(db, target) + NewPipeArchiveRecordWriter(db, target).write(source, categories) + db.commit() + } + ZipOutputStream(output).use { zip -> + zip.putNextEntry(ZipEntry("newpipe.db")) + Files.newInputStream(database).use { it.copyTo(zip) } + zip.closeEntry() + } + } finally { + Files.deleteIfExists(database) + Files.deleteIfExists(database.resolveSibling("${database.fileName}-journal")) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt new file mode 100644 index 00000000..8c08762a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt @@ -0,0 +1,179 @@ +package dev.typetype.server.portability + +import java.sql.Connection + +internal class NewPipeArchiveRecordWriter( + private val db: Connection, + private val target: NewPipeArchiveTarget, +) { + fun write(source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS in categories) subscriptions(source) + if (PortabilityCategory.SUBSCRIPTION_GROUPS in categories) groups(source) + if (PortabilityCategory.HISTORY in categories) history(source) + if (PortabilityCategory.PLAYLISTS in categories) playlists(source) + if (PortabilityCategory.PROGRESS in categories) progress(source) + if (PortabilityCategory.SEARCH_HISTORY in categories) searchHistory(source) + if (PortabilityCategory.SAVED_PLAYLISTS in categories) savedPlaylists(source) + } + + private fun subscriptions(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + if (NewPipeProvider.supported(item.channelUrl, target)) ensureSubscription(item.channelUrl, item.name, item.avatarUrl) + } + } + + private fun groups(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val groupId = db.insertId( + "INSERT INTO feed_group(name,icon_id,sort_order) VALUES(?,?,?)", + record.name, + 0, + order++, + ) + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + val membership = child as? PortabilitySubscriptionGroupMembership ?: return@forEachChild + if (!NewPipeProvider.supported(membership.channelUrl, target)) return@forEachChild + val subscriptionId = ensureSubscription(membership.channelUrl, "", "") + db.execute( + "INSERT OR IGNORE INTO feed_group_subscription_join(group_id,subscription_id) VALUES(?,?)", + groupId, + subscriptionId, + ) + } + } + } + + private fun history(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + if (!NewPipeProvider.supported(item.video.url, target)) return@forEach + val streamId = ensureVideo(item.video) + db.execute( + "INSERT OR IGNORE INTO stream_history(stream_id,access_date,repeat_count) VALUES(?,?,1)", + streamId, + item.watchedAt, + ) + } + } + + private fun playlists(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + val playlist = record as? PortabilityPlaylist ?: return@forEach + val id = if (target.pipePipe) { + db.insertId("INSERT INTO playlists(name,thumbnail_url,display_index) VALUES(?,'',?)", playlist.name, order++) + } else { + db.insertId( + "INSERT INTO playlists(name,is_thumbnail_permanent,thumbnail_stream_id,display_index) VALUES(?,0,-1,?)", + playlist.name, + order++, + ) + } + source.forEachChild(PortabilityCategory.PLAYLISTS, playlist.sourceId) { child -> + val item = child as? PortabilityPlaylistVideo ?: return@forEachChild + if (!NewPipeProvider.supported(item.video.url, target)) return@forEachChild + db.execute( + "INSERT OR REPLACE INTO playlist_stream_join(playlist_id,stream_id,join_index) VALUES(?,?,?)", + id, + ensureVideo(item.video), + item.position, + ) + } + } + } + + private fun progress(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.PROGRESS) { record -> + val item = record as PortabilityProgress + if (!NewPipeProvider.supported(item.videoUrl, target)) return@forEach + val streamId = ensureVideo(PortabilityVideo(item.videoUrl)) + db.execute("INSERT OR REPLACE INTO stream_state(stream_id,progress_time) VALUES(?,?)", streamId, item.positionSeconds) + } + } + + private fun searchHistory(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + val item = record as PortabilitySearchHistory + db.execute( + "INSERT INTO search_history(creation_date,service_id,search) VALUES(?,0,?)", + item.searchedAt, + item.term, + ) + } + } + + private fun savedPlaylists(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + val item = record as PortabilitySavedPlaylist + val serviceId = NewPipeProvider.serviceId(item.url) ?: return@forEach + if (!target.pipePipe && serviceId != 0) return@forEach + db.execute( + "INSERT OR IGNORE INTO remote_playlists(service_id,name,url,thumbnail_url,uploader,display_index,stream_count) VALUES(?,?,?,?,?,?,?)", + serviceId, + item.title, + item.url, + item.thumbnailUrl, + item.uploaderName, + order++, + item.streamCount, + ) + } + } + + private fun ensureSubscription(url: String, name: String, avatar: String): Long { + val serviceId = requireNotNull(NewPipeProvider.serviceId(url)) + db.execute( + "INSERT OR IGNORE INTO subscriptions(service_id,url,name,avatar_url,subscriber_count,description,notification_mode) VALUES(?,?,?,?,NULL,NULL,0)", + serviceId, + url, + name, + avatar, + ) + return db.long("SELECT uid FROM subscriptions WHERE service_id=? AND url=?", serviceId, url) + } + + private fun ensureVideo(video: PortabilityVideo): Long { + val serviceId = requireNotNull(NewPipeProvider.serviceId(video.url)) + val columns = if (target.pipePipe) { + "service_id,url,title,stream_type,duration,uploader,uploader_url,thumbnail_url,view_count,textual_upload_date,upload_date,is_upload_date_approximation,is_paid" + } else { + "service_id,url,title,stream_type,duration,uploader,uploader_url,thumbnail_url,view_count,textual_upload_date,upload_date,is_upload_date_approximation" + } + val values = if (target.pipePipe) "?,?,?,?,?,?,?,?,?,NULL,?,0,0" else "?,?,?,?,?,?,?,?,?,NULL,?,0" + db.execute( + "INSERT OR IGNORE INTO streams($columns) VALUES($values)", + serviceId, + video.url, + video.title, + "VIDEO_STREAM", + video.durationSeconds, + video.channelName, + video.channelUrl, + video.thumbnailUrl, + video.viewCount, + video.publishedAt.takeIf { it >= 0L }, + ) + return db.long("SELECT uid FROM streams WHERE service_id=? AND url=?", serviceId, video.url) + } +} + +private fun Connection.execute(sql: String, vararg values: Any?) { + prepareStatement(sql).use { statement -> + values.forEachIndexed { index, value -> statement.setObject(index + 1, value) } + statement.executeUpdate() + } +} + +private fun Connection.insertId(sql: String, vararg values: Any?): Long { + execute(sql, *values) + return long("SELECT last_insert_rowid()") +} + +private fun Connection.long(sql: String, vararg values: Any?): Long = prepareStatement(sql).use { statement -> + values.forEachIndexed { index, value -> statement.setObject(index + 1, value) } + statement.executeQuery().use { rows -> check(rows.next()); rows.getLong(1) } +} From 5af9a1a11ca69e9fdfd608ac7977febfdb8b58aa Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:40 +0200 Subject: [PATCH 36/68] test: cover NewPipe portability formats --- .../NewPipeArchivePortabilityAdapterTest.kt | 140 ++++++++++++++++++ .../NewPipePortabilityAdapterTest.kt | 52 +++++++ 2 files changed, 192 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt new file mode 100644 index 00000000..07226334 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt @@ -0,0 +1,140 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.sql.DriverManager +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream +import java.util.zip.ZipOutputStream + +class NewPipeArchivePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `newpipe archive streams all supported tables`() { + val input = input(9) + val adapter = NewPipePortabilityAdapter() + val spool = PortabilitySpool.create(directory) + + assertEquals(PortabilityFormat.NEW_PIPE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(1L, spool.counts()[PortabilityCategory.SEARCH_HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.SAVED_PLAYLISTS]) + spool.delete() + } + + @Test + fun `pipepipe archive is distinguished by schema version`() { + val input = input(901) + val adapter = PipePipePortabilityAdapter() + + assertEquals(PortabilityFormat.PIPE_PIPE, requireNotNull(adapter.detect(input)).format) + assertEquals(null, NewPipePortabilityAdapter().detect(input)) + } + + @Test + fun `pipepipe export round trips canonical records through its real archive`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://youtube.com/channel/UC1", "One")) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://youtube.com/channel/UC1")) + val video = PortabilityVideo("https://youtube.com/watch?v=video000001", "Video", durationSeconds = 60) + source.write(PortabilityHistory(video, 10)) + source.write(PortabilityPlaylist("local", "Local")) + source.write(PortabilityPlaylistVideo("local", 0, video)) + source.write(PortabilityProgress(video.url, 20)) + source.write(PortabilitySearchHistory("query", 30)) + source.write(PortabilitySavedPlaylist("remote", "https://youtube.com/playlist?list=PL1", "Remote")) + val output = directory.resolve("pipepipe-export.zip") + val categories = source.categories() + + Files.newOutputStream(output).use { + PipePipePortabilityAdapter().encode(source, it, categories) + } + assertValidRoomArchive(output, NewPipeArchiveTarget.PIPE_PIPE) + val input = PortabilityInputFactory.create(output, output.fileName.toString(), "application/zip") + assertEquals("901", PipePipePortabilityAdapter().detect(input)?.formatVersion) + val restored = PortabilitySpool.create(directory) + PipePipePortabilityAdapter().decode(input, restored) + + assertEquals(1L, restored.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, restored.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, restored.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, restored.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, restored.counts()[PortabilityCategory.PROGRESS]) + assertEquals(1L, restored.counts()[PortabilityCategory.SEARCH_HISTORY]) + assertEquals(1L, restored.counts()[PortabilityCategory.SAVED_PLAYLISTS]) + source.delete() + restored.delete() + } + + private fun assertValidRoomArchive(archive: Path, target: NewPipeArchiveTarget) { + val database = directory.resolve("validated-${target.databaseVersion}.db") + ZipInputStream(Files.newInputStream(archive)).use { zip -> + check(zip.nextEntry?.name == "newpipe.db") + Files.newOutputStream(database).use(zip::copyTo) + } + DriverManager.getConnection("jdbc:sqlite:$database").use { sqlite -> + sqlite.createStatement().use { statement -> + statement.executeQuery("PRAGMA user_version").use { + assertEquals(target.databaseVersion, it.getInt(1)) + } + statement.executeQuery("PRAGMA integrity_check").use { + assertEquals("ok", it.getString(1)) + } + statement.executeQuery("PRAGMA foreign_key_check").use { + assertEquals(false, it.next()) + } + statement.executeQuery("SELECT identity_hash FROM room_master_table WHERE id = 42").use { + assertEquals(target.identityHash, it.getString(1)) + } + } + } + } + + private fun input(version: Int): PortabilityInput { + val db = directory.resolve("source-$version.db") + DriverManager.getConnection("jdbc:sqlite:$db").use { sqlite -> + sqlite.createStatement().use { statement -> + statement.execute("PRAGMA user_version = $version") + statement.execute("CREATE TABLE subscriptions(uid INTEGER, service_id INTEGER, url TEXT, name TEXT, avatar_url TEXT)") + statement.execute("CREATE TABLE streams(uid INTEGER, url TEXT, title TEXT, duration INTEGER, uploader TEXT, uploader_url TEXT, thumbnail_url TEXT)") + statement.execute("CREATE TABLE stream_history(stream_id INTEGER, access_date INTEGER)") + statement.execute("CREATE TABLE stream_state(stream_id INTEGER, progress_time INTEGER)") + statement.execute("CREATE TABLE playlists(uid INTEGER, name TEXT, display_index INTEGER)") + statement.execute("CREATE TABLE playlist_stream_join(playlist_id INTEGER, stream_id INTEGER, join_index INTEGER)") + statement.execute("CREATE TABLE remote_playlists(uid INTEGER, url TEXT, name TEXT, thumbnail_url TEXT, uploader TEXT, stream_count INTEGER, display_index INTEGER)") + statement.execute("CREATE TABLE feed_group(uid INTEGER, name TEXT, sort_order INTEGER)") + statement.execute("CREATE TABLE feed_group_subscription_join(group_id INTEGER, subscription_id INTEGER)") + statement.execute("CREATE TABLE search_history(id INTEGER, search TEXT, creation_date INTEGER)") + statement.execute("INSERT INTO subscriptions VALUES (1, 0, 'https://youtube.com/channel/UC1', 'One', 'avatar')") + statement.execute("INSERT INTO streams VALUES (1, 'https://youtube.com/watch?v=v1', 'Video', 60, 'One', 'https://youtube.com/channel/UC1', 'thumb')") + statement.execute("INSERT INTO stream_history VALUES (1, 10)") + statement.execute("INSERT INTO stream_state VALUES (1, 20)") + statement.execute("INSERT INTO playlists VALUES (1, 'Local', 0)") + statement.execute("INSERT INTO playlist_stream_join VALUES (1, 1, 0)") + statement.execute("INSERT INTO remote_playlists VALUES (2, 'https://youtube.com/playlist?list=PL1', 'Remote', 'thumb', 'One', 1, 0)") + statement.execute("INSERT INTO feed_group VALUES (1, 'News', 0)") + statement.execute("INSERT INTO feed_group_subscription_join VALUES (1, 1)") + statement.execute("INSERT INTO search_history VALUES (1, 'query', 30)") + } + } + val archive = directory.resolve("backup-$version.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("newpipe.db")) + Files.newInputStream(db).use { it.copyTo(output) } + output.closeEntry() + } + return PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt new file mode 100644 index 00000000..a64d2213 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class NewPipePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter streams supported subscriptions and reports unsupported services`() { + val file = directory.resolve("newpipe.json") + Files.writeString( + file, + """{"subscriptions":[{"service_id":0,"url":"https://www.youtube.com/channel/UC1","name":"One"},{"service_id":1,"url":"https://soundcloud.com/two","name":"Two"}],"app_version":"0.28.0","app_version_int":1000}""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = NewPipePortabilityAdapter() + + assertEquals(PortabilityFormat.NEW_PIPE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 1L), spool.counts()) + assertEquals("unsupported_subscription_provider", spool.issues().single().code) + + val output = directory.resolve("export.zip") + Files.newOutputStream(output).use { adapter.encode(spool, it, setOf(PortabilityCategory.SUBSCRIPTIONS)) } + val restored = PortabilitySpool.create(directory) + adapter.decode(PortabilityInputFactory.create(output, "export.zip", "application/zip"), restored) + assertEquals(1L, restored.counts()[PortabilityCategory.SUBSCRIPTIONS]) + restored.delete() + spool.delete() + } + + @Test + fun `adapter preserves an empty subscription section`() { + val file = directory.resolve("empty.json") + Files.writeString(file, """{"subscriptions":[],"app_version":"0.28.0"}""") + val spool = PortabilitySpool.create(directory) + + NewPipePortabilityAdapter().decode( + PortabilityInputFactory.create(file, file.fileName.toString(), "application/json"), + spool, + ) + + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 0L), spool.counts()) + spool.delete() + } +} From dbee27818086fd568ff8146fa06ad22050ac41a4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:40 +0200 Subject: [PATCH 37/68] feat: add PipePipe portability support --- .../portability/PipePipePortabilityAdapter.kt | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt new file mode 100644 index 00000000..e5bb46fb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.portability + +import java.io.OutputStream + +class PipePipePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.PIPE_PIPE, + 2, + newPipeArchiveCapabilities(), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val version = NewPipeArchiveDatabase.userVersion(input) ?: return null + if (version < PIPE_PIPE_DATABASE_VERSION) return null + return PortabilityDetection(PortabilityFormat.PIPE_PIPE, version.toString(), 100, "PipePipe SQLite schema version") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported PipePipe backup" } + NewPipeArchiveDatabase.read(input) { NewPipeDatabasePortabilityReader.read(it, sink) } + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = super.assessExport(source, categories) + + newPipeArchiveExportIssues(source, categories, NewPipeArchiveTarget.PIPE_PIPE) + + override fun encode(source: PortabilityRecordSource, output: OutputStream, categories: Set) = + NewPipeArchivePortabilityWriter.write(source, output, categories, NewPipeArchiveTarget.PIPE_PIPE) + + private companion object { + const val PIPE_PIPE_DATABASE_VERSION = 900 + } +} + +internal fun archiveImportCapabilities(): Set = setOf( + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, +).mapTo(linkedSetOf()) { category -> + PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) +} From 93d05a8bf48ca02dd676cce5f005a140fa25effe Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:52 +0200 Subject: [PATCH 38/68] feat: detect YouTube Takeout backups --- .../YoutubeTakeoutHtmlPortabilityReader.kt | 46 +++++++++++++++ .../portability/YoutubeTakeoutMappings.kt | 55 ++++++++++++++++++ .../YoutubeTakeoutPortabilityAdapter.kt | 56 +++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt new file mode 100644 index 00000000..e48116cf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +import dev.typetype.server.services.YoutubeTakeoutActivitySignalService +import dev.typetype.server.services.YoutubeTakeoutHistoryParser +import java.io.Reader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +internal object YoutubeTakeoutHtmlPortabilityReader { + fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> + zip.getInputStream(entry).bufferedReader().use { reader -> + readWindows(reader) { html -> writeWindow(html, sink) } + } + } + } + + private fun writeWindow(html: String, sink: PortabilityRecordSink) { + YoutubeTakeoutHistoryParser.parse(html).forEach { sink.write(it.toPortability()) } + val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) + subscriptions.forEach { sink.write(it.toPortability()) } + favorites.forEach { sink.write(it.toPortability()) } + } + + private fun readWindows(reader: Reader, block: (String) -> Unit) { + val buffer = CharArray(READ_CHARS) + val window = StringBuilder(WINDOW_CHARS + READ_CHARS) + while (true) { + val read = reader.read(buffer) + if (read < 0) break + window.append(buffer, 0, read) + if (window.length >= WINDOW_CHARS) { + block(window.toString()) + window.delete(0, window.length - OVERLAP_CHARS) + } + } + if (window.isNotEmpty()) block(window.toString()) + } + + private fun isYoutubeHtml(entry: ZipEntry): Boolean = + entry.name.endsWith(".html", ignoreCase = true) && "youtube" in entry.name.lowercase() + + private const val READ_CHARS = 32 * 1024 + private const val WINDOW_CHARS = 512 * 1024 + private const val OVERLAP_CHARS = 128 * 1024 +} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt new file mode 100644 index 00000000..b335dc2b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.portability + +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistVideoItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.YoutubeTypeTypeMapper + +internal fun SubscriptionItem.toPortability() = PortabilitySubscription( + channelUrl = channelUrl, + name = name, + avatarUrl = avatarUrl, + subscribedAt = subscribedAt, +) + +internal fun HistoryItem.toPortability() = PortabilityHistory( + video = PortabilityVideo( + url = url, + title = title, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + ), + watchedAt = watchedAt, + positionSeconds = progress, +) + +internal fun FavoriteItem.toPortability() = PortabilityFavorite( + video = PortabilityVideo( + url = videoUrl, + title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(videoUrl) }, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(videoUrl) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + viewCount = viewCount, + publishedAt = publishedAt, + ), + favoritedAt = favoritedAt, +) + +internal fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( + url = url, + title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(url) }, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + viewCount = viewCount, + publishedAt = publishedAt, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt new file mode 100644 index 00000000..50e3cbfc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.YOUTUBE_TAKEOUT, + adapterVersion = 1, + capabilities = TAKEOUT_CATEGORIES.mapTo(linkedSetOf()) { category -> + PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) + }, + defaultExtension = "zip", + contentType = "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + val names = archive.names.map(String::lowercase) + val youtubeFiles = names.count { name -> + "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html")) + } + if (youtubeFiles == 0) return null + val hasTakeoutRoot = names.any { it.startsWith("takeout/") } + return PortabilityDetection( + PortabilityFormat.YOUTUBE_TAKEOUT, + null, + if (hasTakeoutRoot) 99 else 90, + "YouTube Takeout CSV or activity files", + ) + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported YouTube Takeout archive" } + ZipFile(input.path.toFile()).use { zip -> + val entries = zip.entries().asSequence().filterNot { it.isDirectory }.toList() + YoutubeTakeoutCsvPortabilityReader.readManifests(zip, entries, sink) + YoutubeTakeoutHtmlPortabilityReader.read(zip, entries, sink) + YoutubeTakeoutCsvPortabilityReader.readContent(zip, entries, sink) + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ): Unit = error("YouTube Takeout export is not available") +} + +private val TAKEOUT_CATEGORIES = setOf( + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, +) From d96ffc4334bd960df08121c1f0b79b55961da8bd Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:52 +0200 Subject: [PATCH 39/68] feat: stream YouTube Takeout CSV records --- .../YoutubeTakeoutCsvPortabilityReader.kt | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt new file mode 100644 index 00000000..14577074 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -0,0 +1,135 @@ +package dev.typetype.server.portability + +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.services.YoutubeTakeoutCsvReader +import dev.typetype.server.services.YoutubeTakeoutRowParser +import dev.typetype.server.services.YoutubeTakeoutSchemaHints +import dev.typetype.server.services.YoutubeTakeoutSystemPlaylist +import java.io.BufferedReader +import java.io.InputStreamReader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +internal object YoutubeTakeoutCsvPortabilityReader { + private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" + + fun readManifests(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + csvEntries(entries).filter(::isPlaylistManifest).forEach { entry -> + sink.markCategory(PortabilityCategory.PLAYLISTS) + forEachRow(zip, entry) { header, row -> + val playlist = YoutubeTakeoutRowParser.parsePlaylist(header, row) ?: return@forEachRow + indexPlaylist(playlist, sink) + if (!isSystemPlaylist(playlist)) { + sink.write(PortabilityPlaylist(playlist.id, playlist.name, playlist.description, playlist.createdAt)) + } + } + } + } + + fun readContent(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + val csv = csvEntries(entries).filterNot(::isPlaylistManifest).toList() + csv.filter(::isSubscriptionFile).forEach { readSubscriptions(zip, it, sink) } + csv.filter(::isPlaylistContent).forEach { readPlaylistItems(zip, it, sink) } + } + + private fun readSubscriptions(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + forEachRow(zip, entry) { header, row -> + val item = YoutubeTakeoutRowParser.parseSubscription(header, row) + if (item == null) sink.invalid(PortabilityCategory.SUBSCRIPTIONS, "subscription") + else sink.write(item.toPortability()) + } + } + + private fun readPlaylistItems(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { + var inferredKey = playlistKeyFromPath(entry.name) + var rowPosition = 0 + forEachRow(zip, entry) { originalHeader, row -> + val hasKey = originalHeader.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + originalHeader.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + val header = if (hasKey || inferredKey == null) originalHeader else listOf("playlist source key") + originalHeader + val values = if (header === originalHeader) row else listOf(requireNotNull(inferredKey)) + row + if (YoutubeTakeoutRowParser.isUnavailablePlaylistItem(header, values)) return@forEachRow + val parsed = YoutubeTakeoutRowParser.parsePlaylistItem(header, values) + if (parsed == null) { + sink.invalid(PortabilityCategory.PLAYLISTS, "playlist item") + return@forEachRow + } + inferredKey = parsed.first + val item = parsed.second.copy(position = parsed.second.position.takeIf { it > 0 } ?: rowPosition) + writePlaylistVideo(parsed.first, item, sink) + rowPosition += 1 + } + } + + private fun writePlaylistVideo(key: String, item: dev.typetype.server.models.PlaylistVideoItem, sink: PortabilityRecordSink) { + when (val resolved = YoutubeTakeoutSystemPlaylist.canonicalKey(key) ?: sink.lookup(PLAYLIST_LOOKUP, key)) { + YoutubeTakeoutSystemPlaylist.WATCH_LATER -> sink.write( + PortabilityWatchLater(item.toPortabilityVideo(), item.addedAt), + ) + YoutubeTakeoutSystemPlaylist.LIKED_VIDEOS -> sink.write( + PortabilityFavorite(item.toPortabilityVideo(), item.addedAt), + ) + null -> { + val sourceId = key.trim() + sink.write(PortabilityPlaylist(sourceId, sourceId)) + sink.write(PortabilityPlaylistVideo(sourceId, item.position, item.toPortabilityVideo(), item.addedAt)) + sink.issue(PortabilityIssue(PortabilityCategory.PLAYLISTS, "playlist_manifest_missing", "A playlist was reconstructed from its item file")) + } + else -> sink.write(PortabilityPlaylistVideo(resolved, item.position, item.toPortabilityVideo(), item.addedAt)) + } + } + + private fun indexPlaylist(item: PlaylistItem, sink: PortabilityRecordSink) { + val resolved = YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) ?: YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) + ?: item.id.ifBlank { item.name } + if (item.id.isNotBlank()) sink.putLookup(PLAYLIST_LOOKUP, item.id, resolved) + playlistAliases(item.name).forEach { sink.putLookup(PLAYLIST_LOOKUP, it, resolved) } + } + + private fun forEachRow(zip: ZipFile, entry: ZipEntry, block: (List, List) -> Unit) { + zip.getInputStream(entry).use { input -> + val reader = BufferedReader(InputStreamReader(input, Charsets.UTF_8)) + var header = emptyList() + YoutubeTakeoutCsvReader.forEach(reader, { header = it }) { row -> block(header, row) } + } + } + + private fun csvEntries(entries: List) = entries.asSequence().filter { + it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() + } + + private fun isPlaylistManifest(entry: ZipEntry): Boolean = fileStem(entry) in setOf("playlists", "oynatma listeleri") + + private fun isSubscriptionFile(entry: ZipEntry): Boolean = fileStem(entry) in SUBSCRIPTION_NAMES + + private fun isPlaylistContent(entry: ZipEntry): Boolean { + val normalized = YoutubeTakeoutSchemaHints.normalize(entry.name) + return "playlist" in normalized || "oynatma list" in normalized + } + + private fun playlistKeyFromPath(path: String): String? = path.substringAfterLast('/').substringBeforeLast('.') + .takeUnless { YoutubeTakeoutSchemaHints.normalize(it) == "playlist items" } + ?.let { YoutubeTakeoutSystemPlaylist.canonicalKey(it) ?: it } + + private fun fileStem(entry: ZipEntry) = YoutubeTakeoutSchemaHints.normalize( + entry.name.substringAfterLast('/').substringBeforeLast('.'), + ) + + private fun isSystemPlaylist(item: PlaylistItem) = + YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null + + private fun playlistAliases(name: String): Set = setOf( + name, + "Videos from $name", + "Videos de $name", + "Vidéos de $name", + "Videos da playlist $name", + "$name videos", + ) + + private fun PortabilityRecordSink.invalid(category: PortabilityCategory, kind: String) = + issue(PortabilityIssue(category, "invalid_takeout_row", "An invalid YouTube Takeout $kind row was skipped")) + + private val SUBSCRIPTION_NAMES = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") +} From 8e9d7f5d1661e0709d1e9979d57d56e16e475938 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:37:53 +0200 Subject: [PATCH 40/68] refactor: share YouTube Takeout parsing --- .../YoutubeTakeoutActivitySignalService.kt | 8 +- .../services/YoutubeTakeoutCsvReader.kt | 87 ++++++++++++++----- .../YoutubeTakeoutPortabilityAdapterTest.kt | 71 +++++++++++++++ 3 files changed, 140 insertions(+), 26 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index c14db660..4846bce3 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -20,13 +20,17 @@ object YoutubeTakeoutActivitySignalService { val normalized = entry.name.lowercase() if (entry.isDirectory || !normalized.endsWith(".html") || !normalized.contains("youtube")) return@forEach val html = zip.getInputStream(entry).bufferedReader().use { it.readText() }.replace("\u00a0", " ") - subscriptions += parseSubscriptions(html) - favorites += parseFavorites(html) + val parsed = parseHtml(html) + subscriptions += parsed.first + favorites += parsed.second } } return subscriptions.distinctBy { it.channelUrl } to favorites.distinctBy { it.videoUrl } } + internal fun parseHtml(html: String): Pair, List> = + parseSubscriptions(html) to parseFavorites(html) + private fun parseSubscriptions(html: String): List { return subscribedRegex.findAll(html).mapNotNull { match -> val url = channelUrlRegex.find(decode(match.groupValues[1]))?.value ?: return@mapNotNull null diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt index 5028a11a..988e3125 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt @@ -1,48 +1,87 @@ package dev.typetype.server.services import java.io.BufferedReader +import java.io.PushbackReader object YoutubeTakeoutCsvReader { fun parse(reader: BufferedReader): Pair, List>> { - val records = parseRecords(reader.readText()).filter { row -> row.any { it.isNotBlank() } } - if (records.isEmpty()) return emptyList() to emptyList() - val header = records.first().mapIndexed { index, value -> - if (index == 0) value.removePrefix("\uFEFF") else value + var header = emptyList() + val rows = mutableListOf>() + forEach(reader, { header = it }) { row -> + require(rows.size < MAX_COLLECTED_ROWS) { "CSV contains too many rows" } + rows += row } - return header to records.drop(1) + return header to rows } - private fun parseRecords(content: String): List> { - val records = mutableListOf>() + fun forEach( + reader: BufferedReader, + onHeader: (List) -> Unit, + onRow: (List) -> Unit, + ) { + var headerSeen = false + parseRecords(reader) { record -> + if (record.none(String::isNotBlank)) return@parseRecords + if (!headerSeen) { + onHeader(record.mapIndexed { index, value -> if (index == 0) value.removePrefix("\uFEFF") else value }) + headerSeen = true + } else { + onRow(record) + } + } + } + + private fun parseRecords(source: BufferedReader, block: (List) -> Unit) { + val reader = PushbackReader(source, 1) val row = mutableListOf() val current = StringBuilder() var quoted = false - var index = 0 - while (index < content.length) { - val c = content[index] + var hasData = false + while (true) { + val value = reader.read() + if (value == -1) break + val c = value.toChar() + hasData = true when { - c == '"' && quoted && content.getOrNull(index + 1) == '"' -> { - current.append(c) - index += 1 + c == '"' && quoted -> { + val next = reader.read() + if (next == '"'.code) current.append('"') else { + quoted = false + if (next != -1) reader.unread(next) + } } - c == '"' -> quoted = quoted.not() + c == '"' -> quoted = true c == ',' && !quoted -> { - row += current.toString().trim() - current.clear() + row.addField(current) } (c == '\n' || c == '\r') && !quoted -> { - if (c == '\r' && content.getOrNull(index + 1) == '\n') index += 1 - row += current.toString().trim() - records += row.toList() + if (c == '\r') { + val next = reader.read() + if (next != '\n'.code && next != -1) reader.unread(next) + } + row.addField(current) + block(row.toList()) row.clear() - current.clear() + hasData = false } else -> current.append(c) } - index += 1 + require(current.length <= MAX_FIELD_CHARS) { "CSV field is too large" } + require(row.size <= MAX_COLUMNS) { "CSV contains too many columns" } } - row += current.toString().trim() - records += row.toList() - return records + require(!quoted) { "CSV contains an unterminated quoted field" } + if (hasData || current.isNotEmpty() || row.isNotEmpty()) { + row.addField(current) + block(row.toList()) + } + } + + private fun MutableList.addField(value: StringBuilder) { + add(value.toString().trim()) + value.clear() } + + private const val MAX_COLUMNS = 256 + private const val MAX_FIELD_CHARS = 2 * 1024 * 1024 + private const val MAX_COLLECTED_ROWS = 100_000 } diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt new file mode 100644 index 00000000..84fb2371 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -0,0 +1,71 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter streams takeout categories and keeps playlist order`() { + val archive = directory.resolve("takeout.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube and YouTube Music/subscriptions/subscriptions.csv", + "Channel Id,Channel Url,Channel Title\nUC123456789012,https://youtube.com/channel/UC123456789012,Channel\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/playlists.csv", + "Playlist ID,Playlist Title\nPL123456789,Imported\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Videos de Imported.csv", + "Video ID,Video Title,Video Added Timestamp\nvideo000001,First,2026-01-02T00:00:00Z\nvideo000002,Second,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Watch later.csv", + "Video ID,Video Title\nwatch000001,Later\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Liked videos.csv", + "Video ID,Video Title\nliked000001,Liked\n", + ) + output.entry( + "Takeout/My Activity/YouTube/watch-history.html", + "You watched Seen
1 Jan 2026, 12:00:00 CET
", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = YoutubeTakeoutPortabilityAdapter() + + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + val positions = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> + positions += (record as PortabilityPlaylistVideo).position + } + assertEquals(listOf(0, 1), positions) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} From 303270b260a96105975c75096970f1d078d8ceb3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:38:14 +0200 Subject: [PATCH 41/68] feat: expose asynchronous portability routes --- .../server/routes/PortabilityJobRoutes.kt | 67 +++++++++++++++++ .../server/routes/PortabilityRouteSupport.kt | 44 +++++++++++ .../server/routes/PortabilityRoutes.kt | 73 +++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt new file mode 100644 index 00000000..a93d672f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.routes + +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityImportRequest +import dev.typetype.server.services.AuthService +import io.ktor.http.ContentDisposition +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.call +import io.ktor.server.request.receive +import io.ktor.server.response.header +import io.ktor.server.response.respond +import io.ktor.server.response.respondFile +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +internal fun Route.portabilityJobRoutes(engine: PortabilityEngine, authService: AuthService) { + get("/portability/jobs/{id}") { withJobAccount(authService) { owner, id -> call.respond(engine.snapshot(owner, id)) } } + get("/portability/jobs/{id}/report") { + withJobAccount(authService) { owner, id -> call.respond(engine.report(owner, id)) } + } + post("/portability/jobs/{id}/apply") { + withJobAccount(authService) { owner, id -> + call.respond(HttpStatusCode.Accepted, engine.applyImport(owner, id, call.receive())) + } + } + post("/portability/jobs/{id}/cancel") { + withJobAccount(authService) { owner, id -> call.respond(engine.cancel(owner, id)) } + } + get("/portability/jobs/{id}/artifact") { + withJobAccount(authService) { owner, id -> + val artifact = engine.artifact(owner, id) + call.response.header( + HttpHeaders.ContentDisposition, + ContentDisposition.Attachment + .withParameter(ContentDisposition.Parameters.FileName, artifact.fileName.toString()) + .toString(), + ) + call.respondFile(artifact.toFile()) + } + } + delete("/portability/jobs/{id}") { + withJobAccount(authService) { owner, id -> + engine.delete(owner, id) + call.respond(HttpStatusCode.NoContent) + } + } +} + +private suspend fun io.ktor.server.routing.RoutingContext.withJobAccount( + authService: AuthService, + block: suspend (String, String) -> Unit, +) { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> + val id = call.parameters["id"] + if (id == null) { + call.respondPortabilityError(IllegalArgumentException("Missing portability job id")) + } else { + runCatching { block(owner, id) } + .onFailure { call.respondPortabilityError(it as? Exception ?: RuntimeException(it)) } + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt new file mode 100644 index 00000000..28ac1d61 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.portability.PortabilityContractException +import dev.typetype.server.portability.PortabilityFormat +import dev.typetype.server.portability.PortabilityJobNotFoundException +import dev.typetype.server.portability.PortabilityUploadTooLargeException +import dev.typetype.server.portability.portabilityErrorCode +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +internal fun parsePortabilityFormat(value: String?): PortabilityFormat? { + if (value == null) return null + return PortabilityFormat.entries.firstOrNull { it.wireName == value } + ?: throw IllegalArgumentException("Unsupported portability format") +} + +internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) { + val status = when (error) { + is PortabilityJobNotFoundException -> HttpStatusCode.NotFound + is PortabilityUploadTooLargeException -> HttpStatusCode.PayloadTooLarge + is IllegalStateException -> HttpStatusCode.Conflict + is IllegalArgumentException -> HttpStatusCode.BadRequest + else -> HttpStatusCode.InternalServerError + } + val message = if (error is PortabilityContractException || error is IllegalArgumentException) { + error.message ?: "Invalid portability request" + } else { + "Portability operation failed" + } + respond(status, ErrorResponse(message, portabilityErrorCode(error))) +} + +internal suspend inline fun ApplicationCall.withPortabilityAccount( + userId: String, + crossinline block: suspend (String) -> Unit, +) { + if (userId.startsWith("guest:")) { + respond(HttpStatusCode.Forbidden, ErrorResponse("Guest users do not have portable account data")) + return + } + block(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt new file mode 100644 index 00000000..fad82a0a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.routes + +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityExportRequest +import dev.typetype.server.portability.PortabilityUploadWriter +import dev.typetype.server.services.AuthService +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.PartData +import io.ktor.server.application.call +import io.ktor.server.request.receive +import io.ktor.server.request.receiveMultipart +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import java.nio.file.Files + +fun Route.portabilityRoutes(engine: PortabilityEngine, authService: AuthService) { + get("/portability/formats") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { call.respond(engine.formats()) } + } + } + post("/portability/imports") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> call.uploadImport(engine, owner) } + } + } + post("/portability/exports") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> + runCatching { + val request = call.receive() + engine.startExport(owner, request.format, request.categories) + } + .onSuccess { call.respond(HttpStatusCode.Accepted, it) } + .onFailure { call.respondPortabilityError(it.asException()) } + } + } + } + portabilityJobRoutes(engine, authService) +} + +private suspend fun io.ktor.server.application.ApplicationCall.uploadImport(engine: PortabilityEngine, owner: String) { + val tmp = Files.createTempFile("portability-upload-", ".tmp") + try { + val multipart = receiveMultipart(dev.typetype.server.portability.PortabilityLimits.MAX_UPLOAD_BYTES) + var filename: String? = null + var contentType: String? = null + var files = 0 + while (true) { + val part = multipart.readPart() ?: break + if (part is PartData.FileItem && part.name == "file") { + files += 1 + if (files == 1) { + filename = part.originalFileName ?: "backup" + contentType = part.contentType?.toString() + PortabilityUploadWriter.write(part.provider(), tmp) + } + } + part.release() + } + require(files == 1) { "Exactly one backup file is required" } + val hint = parsePortabilityFormat(request.queryParameters["format"]) + respond(HttpStatusCode.Accepted, engine.startImportPreview(owner, tmp, requireNotNull(filename), contentType, hint)) + } catch (error: Exception) { + respondPortabilityError(error) + } finally { + Files.deleteIfExists(tmp) + } +} + +private fun Throwable.asException(): Exception = this as? Exception ?: RuntimeException(this) From 0271e3f10c1e4dc612b3a3ad1854e170979ac2ba Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:38:15 +0200 Subject: [PATCH 42/68] test: cover portability HTTP workflows --- .../typetype/server/PortabilityRoutesTest.kt | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt new file mode 100644 index 00000000..8f86d003 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt @@ -0,0 +1,165 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.portability.NewPipePortabilityAdapter +import dev.typetype.server.portability.PortabilityCategory +import dev.typetype.server.portability.PortabilityDataPort +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityExportRequest +import dev.typetype.server.portability.PortabilityFormat +import dev.typetype.server.portability.PortabilityImportRequest +import dev.typetype.server.portability.PortabilityJobSnapshot +import dev.typetype.server.portability.PortabilityJobState +import dev.typetype.server.portability.PortabilityJobStore +import dev.typetype.server.portability.PortabilityRecordSink +import dev.typetype.server.portability.PortabilityRecordSource +import dev.typetype.server.portability.PortabilityRegistry +import dev.typetype.server.portability.PortabilitySubscription +import dev.typetype.server.routes.portabilityRoutes +import dev.typetype.server.services.AuthService +import io.ktor.client.call.body +import io.ktor.client.request.delete +import io.ktor.client.request.forms.MultiPartFormDataContent +import io.ktor.client.request.forms.formData +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.Headers +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityRoutesTest { + @TempDir + lateinit var directory: Path + + @Test + fun `authenticated account can preview apply export and delete`() = testApplication { + val engine = engine() + application { + install(ContentNegotiation) { json(CacheJson) } + routing { portabilityRoutes(engine, AuthService.fixed("owner")) } + } + + val formats = client.get("/portability/formats") { authorize() } + assertEquals(HttpStatusCode.OK, formats.status) + + val upload = client.post("/portability/imports") { + authorize() + setBody( + MultiPartFormDataContent( + formData { + append( + "file", + """{"subscriptions":[],"app_version":"0.28.0"}""".toByteArray(), + Headers.build { + append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + append(HttpHeaders.ContentDisposition, "filename=newpipe.json") + }, + ) + }, + ), + ) + } + assertEquals(HttpStatusCode.Accepted, upload.status) + val importId = upload.snapshot().id + val preview = awaitState(importId, PortabilityJobState.READY) + assertEquals(0L, preview.preview?.counts?.get("subscriptions")) + assertEquals(0L, preview.progress?.processed) + + val apply = client.post("/portability/jobs/$importId/apply") { + authorize() + contentType(ContentType.Application.Json) + setBody(CacheJson.encodeToString(PortabilityImportRequest.serializer(), PortabilityImportRequest(setOf(PortabilityCategory.SUBSCRIPTIONS)))) + } + assertEquals(HttpStatusCode.Accepted, apply.status) + awaitState(importId, PortabilityJobState.COMPLETED) + + val export = client.post("/portability/exports") { + authorize() + contentType(ContentType.Application.Json) + setBody( + CacheJson.encodeToString( + PortabilityExportRequest.serializer(), + PortabilityExportRequest(PortabilityFormat.NEW_PIPE, setOf(PortabilityCategory.SUBSCRIPTIONS)), + ), + ) + } + assertEquals(HttpStatusCode.Accepted, export.status) + val exportId = export.snapshot().id + awaitState(exportId, PortabilityJobState.COMPLETED) + val report = client.get("/portability/jobs/$exportId/report") { authorize() } + assertEquals(HttpStatusCode.OK, report.status) + val artifact = client.get("/portability/jobs/$exportId/artifact") { authorize() } + assertEquals(HttpStatusCode.OK, artifact.status) + assertEquals(listOf(0x50, 0x4b), artifact.body().take(2).map(Byte::toInt)) + + val deleted = client.delete("/portability/jobs/$importId") { authorize() } + assertEquals(HttpStatusCode.NoContent, deleted.status) + engine.close() + } + + private fun engine() = PortabilityEngine( + PortabilityRegistry(listOf(NewPipePortabilityAdapter())), + RouteDataPort, + PortabilityJobStore(directory.resolve("jobs")), + CoroutineScope(SupervisorJob() + Dispatchers.Default), + ) + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() = + header(HttpHeaders.Authorization, "Bearer test-jwt") + + private suspend fun io.ktor.client.statement.HttpResponse.snapshot(): PortabilityJobSnapshot = + CacheJson.decodeFromString(PortabilityJobSnapshot.serializer(), bodyAsText()) + + private suspend fun io.ktor.server.testing.ApplicationTestBuilder.awaitState( + id: String, + expected: PortabilityJobState, + ): PortabilityJobSnapshot { + repeat(100) { + val response = client.get("/portability/jobs/$id") { authorize() } + val snapshot = CacheJson.decodeFromString(PortabilityJobSnapshot.serializer(), response.bodyAsText()) + if (snapshot.state == expected) return snapshot + delay(10) + } + error("Job did not reach $expected") + } +} + +private object RouteDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) = source.counts().mapKeys { it.key.wireName }.also { result -> + request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1", "One")) + onCategoryComplete(PortabilityCategory.SUBSCRIPTIONS, 1L) + } +} From 27dd93ee966258b9edafddd5a2f8daab3b0d988b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:38:15 +0200 Subject: [PATCH 43/68] feat: wire the portability engine --- src/main/kotlin/dev/typetype/server/Application.kt | 12 ++++++++++++ .../dev/typetype/server/ApplicationRoutes.kt | 14 +++++++++++++- .../dev/typetype/server/routes/UserDataRoutes.kt | 3 +++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 299d304c..05786bc2 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -24,8 +24,14 @@ import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserConfig import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.services.YoutubeRemoteLoginReadinessService +import dev.typetype.server.portability.PortabilityEngineFactory import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopped import io.ktor.server.netty.EngineMain +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import java.nio.file.Files import java.util.UUID fun main(args: Array) = EngineMain.main(args) @@ -83,6 +89,11 @@ fun Application.module() { val downloaderGatewayService = DownloaderGatewayService(downloaderServiceUrl) val openMojiProxyService = OpenMojiProxyService(cache) val internalHealthService = InternalHealthService(cache, downloaderGatewayService, subtitleServiceUrl) + val portabilityEngine = PortabilityEngineFactory.create( + Files.createTempDirectory("typetype-portability-"), + CoroutineScope(SupervisorJob() + Dispatchers.IO), + ) + monitor.subscribe(ApplicationStopped) { portabilityEngine.close() } configurePlugins(authService) installApplicationRoutes( svc = svc, @@ -102,5 +113,6 @@ fun Application.module() { internalHealthService = internalHealthService, restoreService = restoreService, youtubeRemoteBrowserService = youtubeRemoteBrowserService, + portabilityEngine = portabilityEngine, ) } diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 1976d9cf..3a930d92 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -42,6 +42,7 @@ import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService +import dev.typetype.server.portability.PortabilityEngine import io.ktor.server.application.Application import io.ktor.server.plugins.ratelimit.rateLimit import io.ktor.server.routing.routing @@ -64,6 +65,7 @@ internal fun Application.installApplicationRoutes( internalHealthService: InternalHealthService, restoreService: PipePipeBackupImporterService, youtubeRemoteBrowserService: YoutubeRemoteBrowserService, + portabilityEngine: PortabilityEngine, ) { routing { internalObservabilityRoutes(internalHealthService::check) @@ -117,6 +119,16 @@ internal fun Application.installApplicationRoutes( adminBugReportRoutes(authService, svc.bugReportService, gitHubIssueService) avatarRoutes(avatarService, openMojiProxyService, svc.customAvatarService) rateLimit(USER_DATA_ZONE) { youtubeRemoteBrowserRoutes(youtubeRemoteBrowserService, authService) } - rateLimit(USER_DATA_ZONE) { userDataRoutes(svc, authService, profileService, avatarService, svc.bugReportService, restoreService) } + rateLimit(USER_DATA_ZONE) { + userDataRoutes( + svc, + authService, + profileService, + avatarService, + svc.bugReportService, + restoreService, + portabilityEngine, + ) + } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 1ff721c8..9da6de79 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -6,6 +6,7 @@ import dev.typetype.server.services.AvatarService import dev.typetype.server.services.BugReportService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService +import dev.typetype.server.portability.PortabilityEngine import io.ktor.server.routing.Route internal fun Route.userDataRoutes( @@ -15,6 +16,7 @@ internal fun Route.userDataRoutes( avatarService: AvatarService, bugReportService: BugReportService, restoreService: PipePipeBackupImporterService, + portabilityEngine: PortabilityEngine, ) { historyRoutes(svc.historyService, authService, svc.settingsService) subscriptionGroupsRoutes(svc.subscriptionGroupsService, authService) @@ -50,6 +52,7 @@ internal fun Route.userDataRoutes( bugReportRoutes(bugReportService, authService) restoreRoutes(restoreService, authService) typeTypeBackupRoutes(svc.typeTypeBackupService, authService) + portabilityRoutes(portabilityEngine, authService) homeRecommendationRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) homeRecommendationShortsRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) } From e81aa4c6fdedc5cd473f3ac217c837f9ebcf1ded Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 17:38:16 +0200 Subject: [PATCH 44/68] docs: define the portability API contract --- openapi.yaml | 14 +++ openapi/components/portability.yaml | 121 +++++++++++++++++++++++ openapi/paths/portability.yaml | 148 ++++++++++++++++++++++++++++ 3 files changed, 283 insertions(+) create mode 100644 openapi/components/portability.yaml create mode 100644 openapi/paths/portability.yaml diff --git a/openapi.yaml b/openapi.yaml index 37be9bbe..849c1c99 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -17,6 +17,7 @@ tags: - name: youtube-session - name: user-data - name: rss + - name: portability paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } @@ -56,6 +57,14 @@ paths: /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } + /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats } + /portability/imports: { $ref: ./openapi/paths/portability.yaml#/PortabilityImports } + /portability/exports: { $ref: ./openapi/paths/portability.yaml#/PortabilityExports } + /portability/jobs/{id}: { $ref: ./openapi/paths/portability.yaml#/PortabilityJob } + /portability/jobs/{id}/apply: { $ref: ./openapi/paths/portability.yaml#/PortabilityApply } + /portability/jobs/{id}/cancel: { $ref: ./openapi/paths/portability.yaml#/PortabilityCancel } + /portability/jobs/{id}/report: { $ref: ./openapi/paths/portability.yaml#/PortabilityReport } + /portability/jobs/{id}/artifact: { $ref: ./openapi/paths/portability.yaml#/PortabilityArtifact } /blocked/keywords: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeywords } /blocked/keywords/{keyword}: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeyword } /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } @@ -163,6 +172,11 @@ components: SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } + PortabilityAdapterDescriptor: { $ref: ./openapi/components/portability.yaml#/PortabilityAdapterDescriptor } + PortabilityJobSnapshot: { $ref: ./openapi/components/portability.yaml#/PortabilityJobSnapshot } + PortabilityJobReport: { $ref: ./openapi/components/portability.yaml#/PortabilityJobReport } + PortabilityImportRequest: { $ref: ./openapi/components/portability.yaml#/PortabilityImportRequest } + PortabilityExportRequest: { $ref: ./openapi/components/portability.yaml#/PortabilityExportRequest } BlockedKeywordItem: { $ref: ./openapi/components/user-backup.yaml#/BlockedKeywordItem } AdminSettingsItem: { $ref: ./openapi/components/access-control.yaml#/AdminSettingsItem } AllowedChannelItem: { $ref: ./openapi/components/access-control.yaml#/AllowedChannelItem } diff --git a/openapi/components/portability.yaml b/openapi/components/portability.yaml new file mode 100644 index 00000000..4309bc1b --- /dev/null +++ b/openapi/components/portability.yaml @@ -0,0 +1,121 @@ +PortabilityFormat: + type: string + enum: [typetype, pipepipe, newpipe, invidious, piped, libretube, viewtube, materialious, youtube-local, flow, skytube, grayjay, youtube-takeout, opml] +PortabilityCategory: + type: string + enum: [subscriptions, subscriptionGroups, history, playlists, watchLater, favorites, progress, searchHistory, savedPlaylists, settings, contentFilters] +PortabilityCapability: + type: object + required: [category, directions, fidelity] + properties: + category: { $ref: '#/PortabilityCategory' } + directions: + type: array + uniqueItems: true + items: { type: string, enum: [import, export] } + fidelity: { type: string, enum: [complete, partial] } +PortabilityAdapterDescriptor: + type: object + required: [format, adapterVersion, capabilities, defaultExtension, contentType] + properties: + format: { $ref: '#/PortabilityFormat' } + adapterVersion: { type: integer, minimum: 1 } + capabilities: + type: array + uniqueItems: true + items: { $ref: '#/PortabilityCapability' } + defaultExtension: { type: string } + contentType: { type: string } +PortabilityIssue: + type: object + required: [category, code, message, count] + properties: + category: + allOf: [{ $ref: '#/PortabilityCategory' }] + nullable: true + code: { type: string } + message: { type: string } + count: { type: integer, format: int64, minimum: 1 } +PortabilityDetection: + type: object + required: [format, formatVersion, adapterVersion, confidence, evidence] + properties: + format: { $ref: '#/PortabilityFormat' } + formatVersion: { type: string, nullable: true } + adapterVersion: { type: integer, minimum: 1 } + confidence: { type: integer, minimum: 1, maximum: 100 } + evidence: { type: string } +PortabilityPreview: + type: object + required: [detection, counts, duplicates, issues] + properties: + detection: { $ref: '#/PortabilityDetection' } + counts: + type: object + additionalProperties: { type: integer, format: int64, minimum: 0 } + duplicates: { type: integer, format: int64, minimum: 0 } + issues: + type: array + items: { $ref: '#/PortabilityIssue' } +PortabilityJobSnapshot: + type: object + required: [id, kind, state, createdAt, updatedAt] + properties: + id: { type: string, format: uuid } + kind: { type: string, enum: [import, export] } + state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } + preview: + allOf: [{ $ref: '#/PortabilityPreview' }] + nullable: true + result: + type: object + nullable: true + additionalProperties: { type: integer, format: int64, minimum: 0 } + progress: + allOf: [{ $ref: '#/PortabilityJobProgress' }] + nullable: true + errorCode: { type: string, nullable: true } +PortabilityJobProgress: + type: object + required: [phase, unit, processed, total] + properties: + phase: { type: string, enum: [analyzing, collecting, applying, encoding] } + unit: { type: string, enum: [records, categories, bytes] } + processed: { type: integer, format: int64, minimum: 0 } + total: { type: integer, format: int64, minimum: 0, nullable: true } +PortabilityJobReport: + type: object + required: [id, state] + properties: + id: { type: string, format: uuid } + state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } + preview: + allOf: [{ $ref: '#/PortabilityPreview' }] + nullable: true + result: + type: object + nullable: true + additionalProperties: { type: integer, format: int64, minimum: 0 } + errorCode: { type: string, nullable: true } +PortabilityImportRequest: + type: object + required: [categories] + properties: + categories: + type: array + minItems: 1 + uniqueItems: true + items: { $ref: '#/PortabilityCategory' } + duplicatePolicy: { type: string, enum: [skip, replace], default: skip } +PortabilityExportRequest: + type: object + required: [format, categories] + properties: + format: { $ref: '#/PortabilityFormat' } + categories: + type: array + minItems: 1 + uniqueItems: true + items: { $ref: '#/PortabilityCategory' } diff --git a/openapi/paths/portability.yaml b/openapi/paths/portability.yaml new file mode 100644 index 00000000..060de475 --- /dev/null +++ b/openapi/paths/portability.yaml @@ -0,0 +1,148 @@ +PortabilityFormats: + get: + tags: [portability] + summary: List supported portability adapters + responses: + '200': + description: Adapter capabilities and fidelity + content: + application/json: + schema: + type: array + items: { $ref: ../components/portability.yaml#/PortabilityAdapterDescriptor } + '401': { description: Missing or invalid token } + '403': { description: Guest account } +PortabilityImports: + post: + tags: [portability] + summary: Upload and analyze an account backup + parameters: + - name: format + in: query + required: false + description: Optional explicit source format. The engine auto-detects when omitted. + schema: { $ref: ../components/portability.yaml#/PortabilityFormat } + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: { type: string, format: binary } + responses: + '202': + description: Analysis job created + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Invalid or unrecognized backup } + '401': { description: Missing or invalid token } + '413': { description: Upload exceeds the configured limit } +PortabilityExports: + post: + tags: [portability] + summary: Create an account export + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityExportRequest } + responses: + '202': + description: Export job created + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Unsupported format or category } + '401': { description: Missing or invalid token } +PortabilityJob: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Read an owned portability job + responses: + '200': + description: Current job state + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } + delete: + tags: [portability] + summary: Delete an owned portability job and its temporary files + responses: + '204': { description: Job deleted } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } +PortabilityApply: + parameters: + - $ref: '#/JobId' + post: + tags: [portability] + summary: Apply selected previewed categories + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityImportRequest } + responses: + '202': + description: Transactional import started + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Invalid category selection } + '409': { description: Job is not ready } +PortabilityCancel: + parameters: + - $ref: '#/JobId' + post: + tags: [portability] + summary: Cancel an active portability job + responses: + '200': + description: Cancelled or already terminal job + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '404': { description: Job not found for this account } +PortabilityReport: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Read the durable result and compatibility report for an owned job + responses: + '200': + description: Current or final portability report + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobReport } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } +PortabilityArtifact: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Download a completed export artifact + responses: + '200': + description: Generated export + headers: + Content-Disposition: + schema: { type: string } + content: + application/octet-stream: + schema: { type: string, format: binary } + '404': { description: Job not found for this account } + '409': { description: Export is not complete } +JobId: + name: id + in: path + required: true + schema: { type: string, format: uuid } From 8f3bd7dca755b40a6bea128141cf0b980c66348e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 18:42:53 +0200 Subject: [PATCH 45/68] feat: expose portability job diagnostics --- openapi/components/portability.yaml | 4 +++ .../server/portability/PortabilityEngine.kt | 8 ++++-- .../server/portability/PortabilityJob.kt | 28 +++++++++++++++++-- .../portability/PortabilityJobModels.kt | 4 +++ .../server/portability/PortabilityJobStore.kt | 4 +-- .../server/portability/PortabilityRegistry.kt | 6 ++++ .../server/routes/PortabilityRoutes.kt | 8 ++++-- .../typetype/server/PortabilityRoutesTest.kt | 6 +++- .../portability/PortabilityEngineTest.kt | 28 +++++++++++++++++++ 9 files changed, 85 insertions(+), 11 deletions(-) diff --git a/openapi/components/portability.yaml b/openapi/components/portability.yaml index 4309bc1b..05302336 100644 --- a/openapi/components/portability.yaml +++ b/openapi/components/portability.yaml @@ -66,6 +66,7 @@ PortabilityJobSnapshot: state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } createdAt: { type: integer, format: int64 } updatedAt: { type: integer, format: int64 } + requestId: { type: string, nullable: true } preview: allOf: [{ $ref: '#/PortabilityPreview' }] nullable: true @@ -77,6 +78,7 @@ PortabilityJobSnapshot: allOf: [{ $ref: '#/PortabilityJobProgress' }] nullable: true errorCode: { type: string, nullable: true } + errorMessage: { type: string, nullable: true } PortabilityJobProgress: type: object required: [phase, unit, processed, total] @@ -91,6 +93,7 @@ PortabilityJobReport: properties: id: { type: string, format: uuid } state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } + requestId: { type: string, nullable: true } preview: allOf: [{ $ref: '#/PortabilityPreview' }] nullable: true @@ -99,6 +102,7 @@ PortabilityJobReport: nullable: true additionalProperties: { type: integer, format: int64, minimum: 0 } errorCode: { type: string, nullable: true } + errorMessage: { type: string, nullable: true } PortabilityImportRequest: type: object required: [categories] diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index 7a14f1a1..9d4d83a2 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -24,8 +24,9 @@ class PortabilityEngine internal constructor( filename: String, contentType: String?, formatHint: PortabilityFormat? = null, + requestId: String? = null, ): PortabilityJobSnapshot { - val job = store.create(userId, PortabilityJobKind.IMPORT) + val job = store.create(userId, PortabilityJobKind.IMPORT, requestId) val saved = job.directory.resolve("upload") try { Files.move(upload, saved) @@ -41,9 +42,10 @@ class PortabilityEngine internal constructor( userId: String, format: PortabilityFormat, categories: Set, + requestId: String? = null, ): PortabilityJobSnapshot { require(categories.isNotEmpty()) { "At least one category is required" } - val job = store.create(userId, PortabilityJobKind.EXPORT) + val job = store.create(userId, PortabilityJobKind.EXPORT, requestId) job.task = scope.launch { export(job, format, categories) } return job.snapshot() } @@ -184,7 +186,7 @@ class PortabilityEngine internal constructor( job.tryTransition(ACTIVE_STATES, PortabilityJobState.CANCELLED) throw error } catch (error: Exception) { - job.tryTransition(ACTIVE_STATES, PortabilityJobState.FAILED, errorCode = portabilityErrorCode(error)) + job.fail(error) } } diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt index 53148d65..100ecf63 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -1,6 +1,7 @@ package dev.typetype.server.portability import kotlinx.coroutines.Job +import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.atomic.AtomicReference @@ -10,11 +11,12 @@ internal class PortabilityJob( val ownerId: String, val kind: PortabilityJobKind, val directory: Path, + val requestId: String?, private val clock: () -> Long, ) { val createdAt = clock() private val value = AtomicReference( - PortabilityJobSnapshot(id, kind, PortabilityJobState.QUEUED, createdAt, createdAt), + PortabilityJobSnapshot(id, kind, PortabilityJobState.QUEUED, createdAt, createdAt, requestId), ) @Volatile var task: Job? = null @@ -26,7 +28,7 @@ internal class PortabilityJob( fun snapshot(): PortabilityJobSnapshot = value.get() fun report(): PortabilityJobReport = value.get().let { - PortabilityJobReport(it.id, it.state, it.preview, it.result, it.errorCode) + PortabilityJobReport(it.id, it.state, it.requestId, it.preview, it.result, it.errorCode, it.errorMessage) } fun isTerminal(): Boolean = value.get().state in TERMINAL_STATES @@ -50,6 +52,7 @@ internal class PortabilityJob( preview: PortabilityPreview? = value.get().preview, result: Map? = value.get().result, errorCode: String? = null, + errorMessage: String? = null, ) { while (true) { val current = value.get() @@ -61,6 +64,7 @@ internal class PortabilityJob( result = result, progress = current.progress, errorCode = errorCode, + errorMessage = errorMessage, ) if (value.compareAndSet(current, next)) return } @@ -70,15 +74,32 @@ internal class PortabilityJob( expected: Set, state: PortabilityJobState, errorCode: String? = null, + errorMessage: String? = null, ): Boolean { while (true) { val current = value.get() if (current.state !in expected) return false - val next = current.copy(state = state, updatedAt = clock(), errorCode = errorCode) + val next = current.copy( + state = state, + updatedAt = clock(), + errorCode = errorCode, + errorMessage = errorMessage, + ) if (value.compareAndSet(current, next)) return true } } + fun fail(error: Exception) { + val code = portabilityErrorCode(error) + logger.error("Portability job failed jobId={} requestId={} code={}", id, requestId ?: "none", code, error) + tryTransition( + PortabilityJobState.entries.toSet() - TERMINAL_STATES, + PortabilityJobState.FAILED, + errorCode = code, + errorMessage = portabilityErrorMessage(error), + ) + } + fun delete() { task?.cancel() spool?.delete() @@ -89,6 +110,7 @@ internal class PortabilityJob( } private companion object { + val logger = LoggerFactory.getLogger(PortabilityJob::class.java) val TERMINAL_STATES = setOf( PortabilityJobState.COMPLETED, PortabilityJobState.FAILED, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt index d13b3ac5..97fffcfa 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt @@ -85,19 +85,23 @@ data class PortabilityJobSnapshot( val state: PortabilityJobState, val createdAt: Long, val updatedAt: Long, + val requestId: String? = null, val preview: PortabilityPreview? = null, val result: Map? = null, val progress: PortabilityJobProgress? = null, val errorCode: String? = null, + val errorMessage: String? = null, ) @Serializable data class PortabilityJobReport( val id: String, val state: PortabilityJobState, + val requestId: String? = null, val preview: PortabilityPreview? = null, val result: Map? = null, val errorCode: String? = null, + val errorMessage: String? = null, ) @Serializable diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt index 7d1aa187..feb3eae2 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt @@ -16,11 +16,11 @@ internal class PortabilityJobStore( Files.createDirectories(root) } - fun create(ownerId: String, kind: PortabilityJobKind): PortabilityJob { + fun create(ownerId: String, kind: PortabilityJobKind, requestId: String? = null): PortabilityJob { cleanup() val id = UUID.randomUUID().toString() val directory = Files.createDirectory(root.resolve(id)) - return PortabilityJob(id, ownerId, kind, directory, clock).also { jobs[id] = it } + return PortabilityJob(id, ownerId, kind, directory, requestId, clock).also { jobs[id] = it } } fun get(ownerId: String, id: String): PortabilityJob = jobs[id] diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt index 6432c09f..7ffa01a1 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt @@ -70,3 +70,9 @@ internal fun portabilityErrorCode(error: Exception): String = when (error) { is IllegalArgumentException -> "portability_invalid_input" else -> "portability_failed" } + +internal fun portabilityErrorMessage(error: Exception): String = when (error) { + is PortabilityContractException, is IllegalArgumentException -> + error.message ?: "Invalid portability data" + else -> "Portability operation failed" +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt index fad82a0a..ae05d946 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt @@ -3,6 +3,7 @@ package dev.typetype.server.routes import dev.typetype.server.portability.PortabilityEngine import dev.typetype.server.portability.PortabilityExportRequest import dev.typetype.server.portability.PortabilityUploadWriter +import dev.typetype.server.requestId import dev.typetype.server.services.AuthService import io.ktor.http.HttpStatusCode import io.ktor.http.content.PartData @@ -31,7 +32,7 @@ fun Route.portabilityRoutes(engine: PortabilityEngine, authService: AuthService) call.withPortabilityAccount(userId = userId) { owner -> runCatching { val request = call.receive() - engine.startExport(owner, request.format, request.categories) + engine.startExport(owner, request.format, request.categories, call.requestId()) } .onSuccess { call.respond(HttpStatusCode.Accepted, it) } .onFailure { call.respondPortabilityError(it.asException()) } @@ -62,7 +63,10 @@ private suspend fun io.ktor.server.application.ApplicationCall.uploadImport(engi } require(files == 1) { "Exactly one backup file is required" } val hint = parsePortabilityFormat(request.queryParameters["format"]) - respond(HttpStatusCode.Accepted, engine.startImportPreview(owner, tmp, requireNotNull(filename), contentType, hint)) + respond( + HttpStatusCode.Accepted, + engine.startImportPreview(owner, tmp, requireNotNull(filename), contentType, hint, requestId()), + ) } catch (error: Exception) { respondPortabilityError(error) } finally { diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt index 8f86d003..babd1b5a 100644 --- a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt @@ -53,6 +53,7 @@ class PortabilityRoutesTest { fun `authenticated account can preview apply export and delete`() = testApplication { val engine = engine() application { + installRequestObservability() install(ContentNegotiation) { json(CacheJson) } routing { portabilityRoutes(engine, AuthService.fixed("owner")) } } @@ -62,6 +63,7 @@ class PortabilityRoutesTest { val upload = client.post("/portability/imports") { authorize() + header(REQUEST_ID_HEADER, "portability-route-request") setBody( MultiPartFormDataContent( formData { @@ -78,7 +80,9 @@ class PortabilityRoutesTest { ) } assertEquals(HttpStatusCode.Accepted, upload.status) - val importId = upload.snapshot().id + val uploaded = upload.snapshot() + assertEquals("portability-route-request", uploaded.requestId) + val importId = uploaded.id val preview = awaitState(importId, PortabilityJobState.READY) assertEquals(0L, preview.preview?.counts?.get("subscriptions")) assertEquals(0L, preview.progress?.processed) diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt index b7b76666..4df0f853 100644 --- a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt @@ -55,6 +55,28 @@ class PortabilityEngineTest { engine.close() } + @Test + fun `failed job keeps safe diagnostics for support`() = runBlocking { + val engine = engine(FakeDataPort(), FailingAdapter()) + val upload = directory.resolve("invalid.json") + Files.writeString(upload, "fixture") + + val started = engine.startImportPreview( + "owner", + upload, + "invalid.json", + "application/json", + requestId = "request-portability-test", + ) + val failed = awaitState(engine, "owner", started.id, PortabilityJobState.FAILED) + + assertEquals("request-portability-test", failed.requestId) + assertEquals("portability_invalid_input", failed.errorCode) + assertEquals("Unsupported backup version", failed.errorMessage) + assertEquals(failed.requestId, engine.report("owner", started.id).requestId) + engine.close() + } + @Test fun `cancelled analysis stops before its files can be deleted`() = runBlocking { val engine = engine(FakeDataPort(), SlowAdapter()) @@ -117,6 +139,12 @@ private class SlowAdapter : PortabilityAdapter by FakeAdapter() { } } +private class FailingAdapter : PortabilityAdapter by FakeAdapter() { + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + throw IllegalArgumentException("Unsupported backup version") + } +} + private open class FakeAdapter : PortabilityAdapter { override val descriptor = PortabilityAdapterDescriptor( PortabilityFormat.NEW_PIPE, From 9f6b810906f6517c60eb76d7c8e79d40e46c9066 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 18:42:59 +0200 Subject: [PATCH 46/68] perf: avoid duplicate portability category writes --- .../kotlin/dev/typetype/server/portability/PortabilitySpool.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt index b64dacc6..24acc1d5 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt @@ -6,6 +6,7 @@ import java.io.Closeable import java.nio.file.Files import java.nio.file.Path import java.sql.Connection +import java.util.EnumSet class PortabilitySpool private constructor( val path: Path, @@ -29,12 +30,14 @@ class PortabilitySpool private constructor( private val getLookup = connection.prepareStatement( "SELECT value FROM lookups WHERE namespace = ? AND lookup_key = ?", ) + private val markedCategories = EnumSet.noneOf(PortabilityCategory::class.java) private var pendingWrites = 0 private var attemptedRecords = 0L private var closed = false override fun markCategory(category: PortabilityCategory) { checkOpen() + if (!markedCategories.add(category)) return markCategory.setString(1, category.wireName) markCategory.executeUpdate() pendingWrites += 1 From 3f7aa892eefb16afedb1296a2ca4e887fc7a79b9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 18:59:16 +0200 Subject: [PATCH 47/68] chore: update server dependencies --- build.gradle.kts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index eeace646..620ef3ae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,9 +25,9 @@ repositories { } dependencies { - implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) + implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") - implementation(platform("io.netty:netty-bom:4.2.16.Final")) + implementation(platform("io.netty:netty-bom:4.2.17.Final")) constraints { implementation("org.jsoup:jsoup:1.23.1") { because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") @@ -43,12 +43,12 @@ dependencies { implementation("io.ktor:ktor-server-status-pages-jvm") implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") - implementation("ch.qos.logback:logback-classic:1.6.1") + implementation("ch.qos.logback:logback-classic:1.6.3") implementation("com.github.Priveetee.PipePipeExtractor:extractor:f156813dd4bbebf3b4dffe541fee6c27ae1dd294") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("org.json:json:20260719") - implementation("com.squareup.okhttp3:okhttp:5.4.0") - implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") + implementation("org.json:json:20260814") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.4.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") implementation("com.zaxxer:HikariCP:7.1.0") From 5af0032dba5d1139db36b4faff895b4e3d23b02f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 20:06:14 +0200 Subject: [PATCH 48/68] fix: stabilize portability format detection --- .../MaterialiousPortabilityAdapter.kt | 2 +- .../portability/TypeTypePortabilityAdapter.kt | 3 ++- .../MaterialiousPortabilityAdapterTest.kt | 13 +++++++++++++ .../TypeTypePortabilityAdapterTest.kt | 18 ++++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt index 44bc7ee4..a7a7db4c 100644 --- a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt +++ b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt @@ -23,7 +23,7 @@ class MaterialiousPortabilityAdapter : PortabilityAdapter { override fun detect(input: PortabilityInput): PortabilityDetection? { if (input.archive != null) return null if (opml.detect(input) != null) { - return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "opml", 85, "Materialious-compatible OPML") + return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "opml", 82, "Materialious-compatible OPML") } val probe = input.probe.decodeToString() if (!probe.trimStart().startsWith("{") || !probe.contains("\"subscriptions\"")) return null diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt index 6fec6714..5d73c058 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt @@ -23,7 +23,8 @@ class TypeTypePortabilityAdapter : PortabilityAdapter { override fun detect(input: PortabilityInput): PortabilityDetection? { if (input.archive != null) return null val probe = input.probe.decodeToString() - if (!probe.contains("\"format\":\"$TYPE_TYPE_BACKUP_FORMAT\"")) return null + val formatMarker = Regex("\"format\"\\s*:\\s*\"${Regex.escape(TYPE_TYPE_BACKUP_FORMAT)}\"") + if (!formatMarker.containsMatchIn(probe)) return null val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) return PortabilityDetection(PortabilityFormat.TYPE_TYPE, version, 100, "TypeType backup marker") } diff --git a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt index 07d0183a..f62491b8 100644 --- a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt @@ -46,4 +46,17 @@ class MaterialiousPortabilityAdapterTest { assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) spool.delete() } + + @Test + fun `generic opml auto detection is not ambiguous with materialious`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val registry = PortabilityRegistry(listOf(OpmlPortabilityAdapter(), MaterialiousPortabilityAdapter())) + + assertEquals(PortabilityFormat.OPML, registry.detect(input).second.format) + } } diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt index a50fa784..104975e6 100644 --- a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt @@ -14,6 +14,24 @@ class TypeTypePortabilityAdapterTest { @TempDir lateinit var directory: Path + @Test + fun `detects formatted TypeType backups`() { + val file = directory.resolve("formatted-typetype.json") + Files.writeString( + file, + """ + { + "format": "typetype-backup", + "version": 1, + "subscriptions": [] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + + assertEquals(PortabilityFormat.TYPE_TYPE, TypeTypePortabilityAdapter().detect(input)?.format) + } + @Test fun `legacy TypeType backup round trips through canonical records`() { val source = PortabilitySpool.create(directory) From e25a1d96615042c8cf1126023f2f64311a45e332 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 24 Aug 2026 01:18:51 +0200 Subject: [PATCH 49/68] fix: bound authenticated SABR preparation --- .../services/AuthenticatedSabrInfoCache.kt | 24 ++++++++++++-- .../services/AuthenticatedSabrInfoService.kt | 16 +++++++--- .../services/AuthenticatedSabrPolicy.kt | 7 ++++ .../services/SabrPlaybackInfoResolver.kt | 3 +- .../YoutubeSessionSabrStreamService.kt | 32 +++++++++++++------ .../services/YoutubeSessionTokenScope.kt | 13 ++++++-- 6 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt index 063e63ce..1772414c 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -1,12 +1,18 @@ package dev.typetype.server.services import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withTimeout import java.time.Duration import java.util.concurrent.ConcurrentHashMap internal class AuthenticatedSabrInfoCache( ttl: Duration = Duration.ofMinutes(5), maxEntries: Int = 256, + private val timeoutMs: Long = AuthenticatedSabrPolicy.INFO_TIMEOUT_MS, ) { private val items = BoundedExpiringCache( maxEntries = maxEntries, @@ -23,9 +29,9 @@ internal class AuthenticatedSabrInfoCache( items.get(key)?.let { return AuthenticatedSabrInfoResult.Ready(it) } val pending = CompletableDeferred() val existing = inFlight.putIfAbsent(key, pending) - if (existing != null) return existing.await() + if (existing != null) return awaitExisting(credentials, videoId, loader, existing) return try { - val result = loader() + val result = withTimeout(timeoutMs) { loader() } if (result is AuthenticatedSabrInfoResult.Ready) items.put(key, result.prepared) pending.complete(result) result @@ -37,6 +43,20 @@ internal class AuthenticatedSabrInfoCache( } } + private suspend fun awaitExisting( + credentials: YoutubeSessionCredentials, + videoId: String, + loader: suspend () -> AuthenticatedSabrInfoResult, + pending: CompletableDeferred, + ): AuthenticatedSabrInfoResult = try { + withTimeout(timeoutMs) { pending.await() } + } catch (error: TimeoutCancellationException) { + throw error + } catch (error: CancellationException) { + if (!currentCoroutineContext().isActive) throw error + getOrLoad(credentials, videoId, loader) + } + private data class Key( val userId: String, val credentialFingerprint: String, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index b31ddb9f..66ecdecc 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -2,7 +2,9 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withContext +import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken @@ -22,7 +24,12 @@ internal class AuthenticatedSabrInfoService( if (userId == null || userId.startsWith("guest:")) return AuthenticatedSabrInfoResult.NotConnected val credentials = youtubeSessionService.connectedCredentials(userId) ?: return AuthenticatedSabrInfoResult.NotConnected - return cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + return try { + cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + } catch (error: TimeoutCancellationException) { + logger.warn("authenticated_sabr_probe event=timeout videoId={}", videoId) + AuthenticatedSabrInfoResult.TimedOut + } } private suspend fun fetchUncached( @@ -32,10 +39,10 @@ internal class AuthenticatedSabrInfoService( return try { val prepared = YoutubeSessionTokenScope.withCredentials(credentials) { withContext(Dispatchers.IO) { - val sessionBinding = visitorDataFetcher() - val token = tokenClient.fetchSession(videoId, sessionBinding) + val sessionBinding = runInterruptible(Dispatchers.IO) { visitorDataFetcher() } + val token = runInterruptible(Dispatchers.IO) { tokenClient.fetchSession(videoId, sessionBinding) } ?: error("Token service did not return authenticated SABR tokens") - val info = probe.fetch(videoId, token.youtubeSessionPoToken()) + val info = runInterruptible(Dispatchers.IO) { probe.fetch(videoId, token.youtubeSessionPoToken()) } SabrPreparedInfo( info = info, initialToken = token, @@ -67,6 +74,7 @@ internal class AuthenticatedSabrInfoService( internal sealed interface AuthenticatedSabrInfoResult { data object NotConnected : AuthenticatedSabrInfoResult data object Failed : AuthenticatedSabrInfoResult + data object TimedOut : AuthenticatedSabrInfoResult data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt new file mode 100644 index 00000000..a0a67f70 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt @@ -0,0 +1,7 @@ +package dev.typetype.server.services + +internal object AuthenticatedSabrPolicy { + const val INFO_TIMEOUT_MS = 15_000L + const val STREAM_TIMEOUT_MS = 20_000L + const val TIMEOUT_CODE = "authenticated_sabr_timeout" +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt index 4486e836..0df44214 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -10,7 +10,7 @@ internal class SabrPlaybackInfoResolver( startTimeMs: Long, ): SabrPreparedInfo? = when (val authenticated = authenticatedInfoService?.fetch(userId, videoId)) { is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared - AuthenticatedSabrInfoResult.Failed -> null + AuthenticatedSabrInfoResult.Failed, AuthenticatedSabrInfoResult.TimedOut -> null AuthenticatedSabrInfoResult.NotConnected, null -> sessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) } @@ -22,6 +22,7 @@ internal class SabrPlaybackInfoResolver( return when (val authenticated = authenticatedInfoService?.fetch(holder.key.userId, holder.key.videoId)) { is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared AuthenticatedSabrInfoResult.Failed, + AuthenticatedSabrInfoResult.TimedOut, AuthenticatedSabrInfoResult.NotConnected, null, -> null diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index ae313935..962039b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -2,6 +2,8 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" @@ -9,17 +11,29 @@ internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access t internal class YoutubeSessionSabrStreamService( private val metadataService: YoutubeSessionStreamService, private val infoService: AuthenticatedSabrInfoService, + private val timeoutMs: Long = AuthenticatedSabrPolicy.STREAM_TIMEOUT_MS, ) { suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { - val metadata = metadataService.getStreamInfo(userId, url) ?: return null - if (metadata !is ExtractionResult.Success) return metadata - val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") - return when (val info = infoService.fetch(userId, videoId)) { - is AuthenticatedSabrInfoResult.Ready -> - ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) - AuthenticatedSabrInfoResult.Failed -> - ExtractionResult.Failure("Authenticated SABR playback unavailable") - AuthenticatedSabrInfoResult.NotConnected -> null + return try { + withTimeout(timeoutMs) { + val metadata = metadataService.getStreamInfo(userId, url) ?: return@withTimeout null + if (metadata !is ExtractionResult.Success) return@withTimeout metadata + val videoId = youtubeVideoId(url) ?: return@withTimeout ExtractionResult.BadRequest("Invalid YouTube URL") + when (val info = infoService.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> + ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) + AuthenticatedSabrInfoResult.Failed -> + ExtractionResult.Failure("Authenticated SABR playback unavailable") + AuthenticatedSabrInfoResult.TimedOut -> + ExtractionResult.Failure( + "Authenticated SABR preparation timed out", + AuthenticatedSabrPolicy.TIMEOUT_CODE, + ) + AuthenticatedSabrInfoResult.NotConnected -> null + } + } + } catch (error: TimeoutCancellationException) { + ExtractionResult.Failure("Authenticated SABR preparation timed out", AuthenticatedSabrPolicy.TIMEOUT_CODE) } } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt index 5c0c3c6f..ba8e0bea 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.ServiceList import java.util.concurrent.Semaphore import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean object YoutubeSessionTokenScope { private const val PUBLIC_PERMITS = 64 @@ -35,11 +37,18 @@ object YoutubeSessionTokenScope { } private suspend fun withPermits(count: Int, block: suspend () -> T): T { - withContext(Dispatchers.IO) { permits.acquire(count) } + val acquired = AtomicBoolean(false) return try { + withContext(Dispatchers.IO) { + if (!permits.tryAcquire(count, PERMIT_ACQUIRE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + error("Timed out waiting for YouTube extraction permits") + } + acquired.set(true) + } block() } finally { - permits.release(count) + if (acquired.get()) permits.release(count) } } + private const val PERMIT_ACQUIRE_TIMEOUT_MS = 15_000L } From f3dc24f6fd99256b8324ae00824d4488a14fb22e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 24 Aug 2026 01:18:56 +0200 Subject: [PATCH 50/68] test: cover authenticated SABR timeouts --- .../AuthenticatedSabrInfoCacheTest.kt | 84 ++++++++++ .../AuthenticatedSabrInfoServiceTest.kt | 12 +- .../services/AuthenticatedSabrTimeoutTest.kt | 145 ++++++++++++++++++ 3 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt new file mode 100644 index 00000000..4c6e2828 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt @@ -0,0 +1,84 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Test +import java.time.Duration + +@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) +class AuthenticatedSabrInfoCacheTest { + @Test + fun `concurrent timeout callers share one bounded load`() = runTest { + val cache = AuthenticatedSabrInfoCache(timeoutMs = 100L) + var loads = 0 + val calls = List(3) { + async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + } + runCurrent() + advanceTimeBy(100L) + runCurrent() + + calls.forEach { call -> + assertInstanceOf(TimeoutCancellationException::class.java, runCatching { call.await() }.exceptionOrNull()) + } + assertEquals(1, loads) + } + + @Test + fun `cancelled leader does not poison a following caller`() = runTest { + val cache = AuthenticatedSabrInfoCache(timeoutMs = 100L) + var loads = 0 + val leader = async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + runCurrent() + leader.cancelAndJoin() + + val follower = async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + runCurrent() + advanceTimeBy(100L) + runCurrent() + + assertInstanceOf(TimeoutCancellationException::class.java, runCatching { follower.await() }.exceptionOrNull()) + assertEquals(2, loads) + } + + @Test + fun `successful concurrent callers share one ready result`() = runTest { + val cache = AuthenticatedSabrInfoCache(Duration.ofMinutes(1), timeoutMs = 100L) + var loads = 0 + val prepared = mockkPrepared() + val calls = List(3) { + async { + cache.getOrLoad(credentials, VIDEO_ID) { + loads++ + advanceTimeBy(50L) + AuthenticatedSabrInfoResult.Ready(prepared) + } + } + } + advanceTimeBy(50L) + runCurrent() + + calls.forEach { call -> assertEquals(prepared, (call.await() as AuthenticatedSabrInfoResult.Ready).prepared) } + assertEquals(1, loads) + } + + private fun mockkPrepared(): SabrPreparedInfo = mockk() + + private companion object { + val credentials = YoutubeSessionCredentials( + userId = "user", + fingerprint = "fingerprint", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + const val VIDEO_ID = "video-id" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt index 9b6e3449..ffdb5440 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -6,7 +6,7 @@ import io.mockk.every import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertThrows @@ -17,7 +17,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo class AuthenticatedSabrInfoServiceTest { @Test - fun `connected account uses one authenticated token pair`() = runTest { + fun `connected account uses one authenticated token pair`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val probe = mockk() @@ -48,7 +48,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `reuses authenticated info for the following playback request`() = runTest { + fun `reuses authenticated info for the following playback request`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val probe = mockk() @@ -73,7 +73,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `guest playback does not inspect connected credentials`() = runTest { + fun `guest playback does not inspect connected credentials`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val service = AuthenticatedSabrInfoService(sessions, tokenClient) @@ -86,7 +86,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `authenticated probe failure is typed and does not mark session used`() = runTest { + fun `authenticated probe failure is typed and does not mark session used`() = runBlocking { val sessions = mockk() val tokenClient = mockk() coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) @@ -119,7 +119,7 @@ class AuthenticatedSabrInfoServiceTest { ) assertThrows(CancellationException::class.java) { - runTest { service.fetch(USER_ID, VIDEO_ID) } + runBlocking { service.fetch(USER_ID, VIDEO_ID) } } } diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt new file mode 100644 index 00000000..549950a8 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt @@ -0,0 +1,145 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +class AuthenticatedSabrTimeoutTest { + @Test + fun `visitor data timeout returns a typed failure`() = runBlocking { + val service = service(visitor = { blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `token timeout returns a typed failure`() = runBlocking { + val service = service(token = { _, _, _ -> blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `probe timeout returns a typed failure`() = runBlocking { + val service = service(probe = { _, _ -> blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `timeout releases credentials for the next request`() = runBlocking { + var calls = 0 + val service = service(visitor = { + calls++ + if (calls == 1) blockingCall() else SESSION_BINDING + }) + + val timedOut = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + val recovered = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, timedOut) + assertEquals(2, calls) + assertTrue(recovered is AuthenticatedSabrInfoResult.Ready, recovered.toString()) + } + + @Test + fun `authenticated stream metadata timeout is typed`() = runTest { + val metadata = mockk() + coEvery { metadata.getStreamInfo(USER_ID, URL) } coAnswers { + delay(60_000L) + error("unreachable") + } + val service = YoutubeSessionSabrStreamService( + metadata, + mockk(relaxed = true), + timeoutMs = 20L, + ) + + val result = withTimeout(1_000L) { service.getStreamInfo(USER_ID, URL) } + + assertTrue(result is ExtractionResult.Failure) + assertEquals(AuthenticatedSabrPolicy.TIMEOUT_CODE, (result as ExtractionResult.Failure).code) + } + + private fun service( + visitor: () -> String = { SESSION_BINDING }, + token: (String, String, Boolean) -> SabrTokenBundle? = { _, _, _ -> sessionToken() }, + probe: (String, YoutubeSessionPoToken) -> YoutubeSabrInfo = { _, _ -> playableInfo() }, + ): AuthenticatedSabrInfoService { + val sessions = mockk() + val tokenClient = mockk() + val probeMock = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials() + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(any(), any(), any()) } answers { + token(arg(0), arg(1), arg(2)) + } + every { probeMock.fetch(any(), any()) } answers { probe(arg(0), arg(1)) } + return AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = visitor, + probe = probeMock, + cache = AuthenticatedSabrInfoCache(timeoutMs = 1_200L), + ) + } + + private fun blockingCall(): Nothing { + Thread.sleep(5_000L) + error("unreachable") + } + + private fun playableInfo(): YoutubeSabrInfo = mockk { + every { formats } returns listOf( + mockk { + every { isAudio } returns true + every { isVideo } returns false + }, + mockk { + every { isAudio } returns false + every { isVideo } returns true + }, + ) + } + + private fun credentials() = YoutubeSessionCredentials( + userId = USER_ID, + fingerprint = "fingerprint", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + + private fun sessionToken() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "public-session-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "public-visitor", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + sessionBinding = SESSION_BINDING, + sessionBoundPoToken = "connected-session-token", + ) + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + const val URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val SESSION_BINDING = "connected-visitor" + } +} From d9a51604b148c5fdcb4cf09836b0773ad15c3af9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 24 Aug 2026 01:19:04 +0200 Subject: [PATCH 51/68] fix: keep SABR watchdog responsive --- .../server/services/SabrDemandAttemptFinisher.kt | 2 ++ .../server/services/SabrDemandWatchdog.kt | 2 +- .../typetype/server/services/SabrPumpLogger.kt | 15 +++++++++++++++ .../services/SabrDemandWatchdogBackoffTest.kt | 2 +- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index e0186344..3e733749 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -26,6 +26,7 @@ internal object SabrDemandAttemptFinisher { if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false holder.clearSegmentDemands() val message = "SABR demand stalled for ${demand.request.summary()}" + SabrPumpLogger.expired(holder, demand.request, recoverable) holder.failTerminal(if (recoverable) sabrRecoverableFailureMessage(message) else message) true } @@ -40,6 +41,7 @@ internal object SabrDemandAttemptFinisher { if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false val current = holder.nextSegmentDemand() ?: return@synchronized false if (!current.matches(request) || holder.segmentDemandIdentity(current) != identity) return@synchronized false + SabrPumpLogger.expired(holder, request, recoverable) fail(holder, request, identity, recoverable) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index aba331af..2a0e3874 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -78,5 +78,5 @@ internal class SabrDemandWatchdog( } private fun nextCheckDelayMs(backoffRemainingMs: Long, futureLiveRequest: Boolean): Long = - maxOf(intervalMs, backoffRemainingMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) + maxOf(intervalMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index c9a759e2..d3230963 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -52,6 +52,21 @@ internal object SabrPumpLogger { ) } + fun expired(holder: SabrSessionHolder, request: SabrSegmentRequest, recoverable: Boolean): Unit { + logger.warn( + "sabr_pump event=demand_expired videoId={} request={} recoverable={} state={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", + holder.key.videoId, + request.summary(), + recoverable, + holder.playbackState(), + holder.session.requestNumber, + holder.session.streamState.getMinBufferedEndMs(), + holder.readerHeadMs(), + holder.readerTailMs(), + holder.session.cachedBytes, + ) + } + fun recovery(holder: SabrSessionHolder, action: SabrDemandRecoveryAction, request: SabrSegmentRequest): Unit { logger.info( "sabr_pump event=demand_recovery videoId={} request={} action={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt index 6607bafb..facd8627 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt @@ -88,7 +88,7 @@ class SabrDemandWatchdogBackoffTest { } runCurrent() - advanceTimeBy(45_900L) + advanceTimeBy(44_900L) runCurrent() assertFalse(job.isCompleted) From 2eefc99c68001a354afb1f383fd91bc2559fa8f2 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 24 Aug 2026 15:52:26 +0200 Subject: [PATCH 52/68] feat: expose parental control capability --- openapi/components/instance.yaml | 4 ++++ .../kotlin/dev/typetype/server/models/InstanceResponse.kt | 1 + .../kotlin/dev/typetype/server/services/InstanceService.kt | 1 + src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt | 3 +++ 4 files changed, 9 insertions(+) diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index a0629e47..c40a33cc 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,6 +20,7 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady + - parentalControlsEnabled - rss properties: name: { type: string, example: TypeType } @@ -50,6 +51,9 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] + parentalControlsEnabled: + type: boolean + description: True when the instance-wide allow-list policy is enabled. rss: $ref: '#/RssInstanceCapability' RssInstanceCapability: diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 55d8916d..133514e6 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,6 +24,7 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, + val parentalControlsEnabled: Boolean = false, val rss: RssInstanceCapability = RssInstanceCapability(), ) diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index 3225e2d6..ab42ccb1 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -48,6 +48,7 @@ class InstanceService( youtubeRemoteLoginEnabled = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginReady = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginUnavailableReason = youtubeRemoteLoginStatus.unavailableReason, + parentalControlsEnabled = settings.accessMode == ACCESS_MODE_ALLOW_LIST, rss = RssInstanceCapability( enabled = settings.rssEnabled && settings.rssPublicBaseUrl != null, maxFeedsPerUser = settings.rssMaxFeedsPerUser, diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index 7d2eeb22..7eca4fc0 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -72,6 +72,7 @@ class InstanceRoutesTest { assertEquals(false, root["oidcEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) + assertEquals(false, root["parentalControlsEnabled"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) val rss = root["rss"]?.jsonObject assertEquals(false, rss?.get("enabled")?.jsonPrimitive?.boolean) @@ -93,6 +94,7 @@ class InstanceRoutesTest { localLoginEnabled = false, oidcAutoRedirect = true, youtubeRemoteLoginEnabled = true, + accessMode = "allow_list", rssEnabled = true, rssPublicBaseUrl = "https://video.example/", rssMaxFeedsPerUser = 4, @@ -123,6 +125,7 @@ class InstanceRoutesTest { assertEquals(true, root["oidcAutoRedirect"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) + assertEquals(true, root["parentalControlsEnabled"]?.jsonPrimitive?.boolean) assertEquals(null, root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) val rss = root["rss"]!!.jsonObject assertEquals(true, rss["enabled"]?.jsonPrimitive?.boolean) From a76441a6724ce8fd1347d351903cd2c5468c905c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 24 Aug 2026 15:52:33 +0200 Subject: [PATCH 53/68] feat: include display context in bug reports --- .../typetype/server/models/BugReportContext.kt | 7 +++++++ .../server/services/BugReportValidation.kt | 11 +++++++++++ .../dev/typetype/server/BugReportRoutesTest.kt | 16 ++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt b/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt index c5e905b9..5f5e45ab 100644 --- a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt +++ b/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt @@ -27,6 +27,13 @@ data class BugReportContextItem( val timestamp: Long, val userAgent: String, val browserLanguage: String, + val viewportWidth: Int? = null, + val viewportHeight: Int? = null, + val screenWidth: Int? = null, + val screenHeight: Int? = null, + val devicePixelRatio: Double? = null, + val online: Boolean? = null, + val timezone: String? = null, val playerState: JsonElement? = null, val crashLogs: List = emptyList(), val apiErrors: List = emptyList(), diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt index b25c9f0f..e19625ff 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt +++ b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt @@ -29,6 +29,17 @@ internal object BugReportValidation { if (context.apiErrors.any { it.endpoint.isBlank() || it.timestamp <= 0 || it.status <= 0 }) { return "Invalid api error entry" } + val dimensions = listOf( + context.viewportWidth, + context.viewportHeight, + context.screenWidth, + context.screenHeight, + ).filterNotNull() + if (dimensions.any { it !in 1..100_000 }) return "Invalid display dimensions" + if (context.devicePixelRatio != null && context.devicePixelRatio !in 0.1..100.0) { + return "Invalid device pixel ratio" + } + if (context.timezone != null && context.timezone.length > 128) return "Invalid timezone" return null } } diff --git a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt b/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt index fa2bf1df..6cf607c4 100644 --- a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt @@ -82,6 +82,22 @@ class BugReportRoutesTest { assertEquals(HttpStatusCode.Created, response.status) } + @Test + fun `POST bug report accepts bounded device diagnostics`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { bugReportRoutes(service, auth) } + } + val response = client.post("/bug-reports") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + contentType(ContentType.Application.Json) + setBody( + """{"category":"ui","description":"Responsive issue","context":{"route":"/settings","timestamp":1774200000000,"userAgent":"Mozilla","browserLanguage":"fr-FR","viewportWidth":390,"viewportHeight":844,"screenWidth":430,"screenHeight":932,"devicePixelRatio":3.0,"online":true,"timezone":"Europe/Paris"}}""", + ) + } + assertEquals(HttpStatusCode.Created, response.status) + } + @Test fun `POST bug report with invalid category returns 400`() = testApplication { application { From 14bb5b9796b191fc321682ab8ec80d9e3f0efc45 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 10:53:21 +0200 Subject: [PATCH 54/68] perf: avoid redundant playback extraction --- .../server/routes/SabrPlaybackHandler.kt | 1 + .../routes/SabrPlaybackAccessValidatorTest.kt | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 0e544d4b..bec077e1 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -128,6 +128,7 @@ internal class SabrPlaybackHandler( } private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { + if (!access.profile.enabled) return true return when (val result = accessValidator.resolve(access.userId, videoId)) { is ExtractionResult.Success -> { val allowed = !access.profile.enabled || diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt index dd1e0683..ec12e743 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt @@ -3,12 +3,46 @@ package dev.typetype.server.routes import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.StreamService +import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.testStreamResponse +import io.ktor.client.request.post +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.call +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.post +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test class SabrPlaybackAccessValidatorTest { + @Test + fun `unrestricted playback does not wait for full stream metadata`() = testApplication { + val store = mockk(relaxed = true) + val streams = mockk() + coEvery { store.fetchInfo("video-id", 0L, cachedFirst = true) } returns null + application { + install(ContentNegotiation) { json() } + val handler = SabrPlaybackHandler(store, streams, null, null, null) + routing { + post("/sabr/playback/{videoId}") { + handler.create(call, call.parameters["videoId"].orEmpty()) + } + } + } + + val response = client.post("/sabr/playback/video-id") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + coVerify(exactly = 0) { streams.getStreamInfo(any()) } + } + @Test fun `uses linked YouTube session even when public metadata is accessible`() = runBlocking { val authenticated = ExtractionResult.Success(testStreamResponse().copy(title = "Authenticated")) From 59ef699d99e201f5101413f8b0d520a81b6b27fc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 15:43:22 +0200 Subject: [PATCH 55/68] feat: stream SABR media responses --- .../server/routes/SabrMediaResponseWriter.kt | 57 +++++++++++++++++++ .../server/SabrMediaResponseWriterTest.kt | 27 ++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt b/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt index b17ad40c..3998300f 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt @@ -5,6 +5,12 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond import io.ktor.server.response.respondBytes +import io.ktor.server.response.respondOutputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import java.io.EOFException +import java.io.InputStream +import java.io.OutputStream internal suspend fun ApplicationCall.respondSabrMediaBytes(mimeType: String, body: ByteArray): Unit { val total = body.size.toLong() @@ -26,3 +32,54 @@ internal suspend fun ApplicationCall.respondSabrMediaBytes(mimeType: String, bod null -> respondBytes(body, containerMime(mimeType), HttpStatusCode.OK) } } + +internal suspend fun ApplicationCall.respondSabrMediaStream( + mimeType: String, + total: Long, + openStream: () -> InputStream, + onOpened: () -> Unit, +): Unit { + response.headers.append(HttpHeaders.CacheControl, "no-store") + response.headers.append(HttpHeaders.AcceptRanges, "bytes") + when (val range = parseAudioOnlyByteRange(request.headers[HttpHeaders.Range], total)) { + is AudioOnlyByteRange.Satisfiable -> { + response.headers.append(HttpHeaders.ContentRange, "bytes ${range.first}-${range.last}/${range.total}") + stream(openStream, onOpened, range.first, range.last - range.first + 1L, HttpStatusCode.PartialContent, mimeType) + } + is AudioOnlyByteRange.Unsatisfiable -> { + response.headers.append(HttpHeaders.ContentRange, "bytes */${range.total}") + respond(HttpStatusCode.RequestedRangeNotSatisfiable) + } + null -> stream(openStream, onOpened, 0L, total, HttpStatusCode.OK, mimeType) + } +} + +private suspend fun ApplicationCall.stream( + openStream: () -> InputStream, + onOpened: () -> Unit, + offset: Long, + length: Long, + status: HttpStatusCode, + mimeType: String, +): Unit = respondOutputStream(containerMime(mimeType), status, length) { + openStream().use { input -> + runInterruptible(Dispatchers.IO) { + input.skipNBytes(offset) + onOpened() + copyExactly(input, length) + } + } +} + +private fun OutputStream.copyExactly(input: InputStream, length: Long) { + val buffer = ByteArray(COPY_BUFFER_SIZE) + var remaining = length + while (remaining > 0L) { + val read = input.read(buffer, 0, minOf(buffer.size.toLong(), remaining).toInt()) + if (read < 0) throw EOFException("SABR media segment ended before its declared length") + write(buffer, 0, read) + remaining -= read + } +} + +private const val COPY_BUFFER_SIZE = 64 * 1024 diff --git a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt b/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt index f11a527e..8d352240 100644 --- a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.routes.respondSabrMediaBytes +import dev.typetype.server.routes.respondSabrMediaStream import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.statement.bodyAsBytes @@ -13,6 +14,8 @@ import io.ktor.server.testing.testApplication import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.util.concurrent.atomic.AtomicInteger class SabrMediaResponseWriterTest { @Test @@ -40,11 +43,33 @@ class SabrMediaResponseWriterTest { assertArrayEquals(byteArrayOf(2, 3, 4, 5), response.bodyAsBytes()) } - private fun ApplicationTestBuilder.installApp(): Unit = application { + @Test + fun `sabr media stream honors byte ranges without buffering the body`() = testApplication { + val opened = AtomicInteger() + installApp(opened) + + val response = client.get("/stream") { header(HttpHeaders.Range, "bytes=3-7") } + + assertEquals(HttpStatusCode.PartialContent, response.status) + assertEquals("bytes 3-7/10", response.headers[HttpHeaders.ContentRange]) + assertEquals("5", response.headers[HttpHeaders.ContentLength]) + assertArrayEquals(byteArrayOf(3, 4, 5, 6, 7), response.bodyAsBytes()) + assertEquals(1, opened.get()) + } + + private fun ApplicationTestBuilder.installApp(opened: AtomicInteger = AtomicInteger()): Unit = application { routing { get("/media") { call.respondSabrMediaBytes("video/mp4; codecs=\"avc1.640028\"", ByteArray(10) { it.toByte() }) } + get("/stream") { + call.respondSabrMediaStream( + "video/mp4; codecs=\"avc1.640028\"", + 10L, + { ByteArrayInputStream(ByteArray(10) { it.toByte() }) }, + opened::incrementAndGet, + ) + } } } } From 960ca717b3ef494da4e5e33c782ea3cc7c34e514 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 15:43:31 +0200 Subject: [PATCH 56/68] perf: stream VOD segments progressively --- .../server/routes/SabrPlaybackHandler.kt | 6 + .../routes/SabrPlaybackWindowBuilder.kt | 20 ++-- .../routes/SabrPlaybackWindowHandler.kt | 18 +-- .../routes/SabrProgressivePlaybackWindow.kt | 70 +++++++++++ .../services/SabrPlaybackMediaFetcher.kt | 110 ++++++++++++++++++ .../services/SabrPlaybackSegmentResult.kt | 8 ++ .../services/SabrPlaybackSessionService.kt | 47 +------- .../server/services/SabrSegmentCache.kt | 3 + .../server/SabrPlaybackGranularRoutesTest.kt | 11 +- .../SabrPlaybackSessionServiceTest.kt | 15 +-- .../server/services/SabrSegmentCacheTest.kt | 27 ++--- 11 files changed, 248 insertions(+), 87 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index bec077e1..6326898a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -14,6 +14,7 @@ import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService +import dev.typetype.server.services.markServed import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -118,6 +119,11 @@ internal class SabrPlaybackHandler( private suspend fun respondSegment(call: ApplicationCall, result: SabrPlaybackSegmentResult): Unit = when (result) { is SabrPlaybackSegmentResult.Ready -> call.respondSabrMediaBytes(result.mimeType, result.bytes) + is SabrPlaybackSegmentResult.Stream -> call.respondSabrMediaStream( + result.mimeType, + result.segment.length.toLong(), + result.segment::openStream, + ) { result.holder.markServed(result.segment, result.generation) } is SabrPlaybackSegmentResult.Retry -> call.respond( HttpStatusCode.Accepted, result.holder.toRetryPlaybackResponse(result.status, RETRY_AFTER_MS), diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt index 82c8113a..a67ee33a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -12,7 +12,6 @@ import dev.typetype.server.services.livePlaybackSnapshot import dev.typetype.server.services.playbackContinuationSequence import dev.typetype.server.services.playbackSegmentStartMs import dev.typetype.server.services.resolvePlaybackStartMs -import dev.typetype.server.services.playbackStartSequence import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat @@ -93,7 +92,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi sequenceOf(video, audio) .filter { it.blockedRequest != null } .minByOrNull { it.coveredEndMs } - private suspend fun buildTrack( holder: SabrSessionHolder, format: YoutubeSabrFormat, @@ -134,6 +132,17 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi } } if (segment == null) { + val progressive = if (activeLive) null else segments.appendProgressiveWindowSegment( + holder, format, seq, expectedStartMs, + ) + if (progressive != null) { + seq = progressive.nextSequence + coveredEndMs = progressive.coveredEndMs + if (coveredEndMs >= goalEndMs) break + blockedRequest = SabrSegmentRequest.media(format, seq) + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + break + } blockedBy = "${format.trackName()}:${format.itag}:$seq pending" blockedRequest = mediaRequest break @@ -169,12 +178,11 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi if (blockedBy == null && coveredEndMs < goalEndMs && !atEnd) { blockedBy = "${format.trackName()}:${format.itag}:$seq window capped" } - val mediaBasePath = SabrPlaybackPaths.mediaBasePath(holder.sessionToken) - val defaultInitUrl = "$mediaBasePath/${format.itag}/init?generation=${holder.activeGeneration()}" return TrackBuildResult( track = SabrPlaybackWindowTrack( mime = format.mimeType.orEmpty(), - initUrl = defaultInitUrl, + initUrl = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/init" + + "?generation=${holder.activeGeneration()}", segments = segments, ), blockedBy = blockedBy, @@ -197,7 +205,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi durationMs = resolvedDurationMs, ) } - private data class TrackBuildResult( val track: SabrPlaybackWindowTrack, val blockedBy: String?, @@ -207,7 +214,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi ) { fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs } - private companion object { const val MAX_SEGMENTS_PER_TRACK = 12 } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt index b25629bf..52d4d2c9 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt @@ -30,7 +30,7 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi sabrSessionStore.startPump(holder) val window = buildWithTargetedPrefetch(holder, request) - if (window.isReady) { + if (holder.canServe(window)) { return call.respond(HttpStatusCode.OK, window.response) } call.respond(HttpStatusCode.Accepted, holder.preparingResponse(request, window)) @@ -68,8 +68,8 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi holder.applyClientPreferences() sabrSessionStore.startPump(holder) val window = buildWithTargetedPrefetch(holder, request) - val status = if (window.isReady) HttpStatusCode.OK else HttpStatusCode.Accepted - call.respond(status, holder.prefetchResponse(request, window)) + val ready = holder.canServe(window) + call.respond(if (ready) HttpStatusCode.OK else HttpStatusCode.Accepted, holder.prefetchResponse(request, window, ready)) } suspend fun segments(call: ApplicationCall, sessionId: String) { @@ -80,7 +80,7 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) holder.applyClientPreferences() val window = windowBuilder.build(holder, request) - if (window.isReady) return call.respond(HttpStatusCode.OK, window.response) + if (holder.canServe(window)) return call.respond(HttpStatusCode.OK, window.response) call.respond(HttpStatusCode.Accepted, holder.preparingResponse(request, window)) } @@ -95,6 +95,9 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi return window } + private fun SabrSessionHolder.canServe(window: SabrPlaybackWindowBuildResult): Boolean = + window.isReady && terminalFailure() == null && networkFailure() == null + private suspend fun SabrSessionHolder.preparingResponse( request: SabrPlaybackWindowRequest, window: SabrPlaybackWindowBuildResult, @@ -122,15 +125,16 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi private suspend fun SabrSessionHolder.prefetchResponse( request: SabrPlaybackWindowRequest, window: SabrPlaybackWindowBuildResult, + ready: Boolean, ): SabrPlaybackPrefetchResponse = SabrPlaybackPrefetchResponse( sessionId = sessionToken, generation = activeGeneration(), - ready = window.isReady, - retryAfterMs = if (window.isReady) null else liveRetryAfterMs(window.blockedRequests), + ready = ready, + retryAfterMs = if (ready) null else liveRetryAfterMs(window.blockedRequests), status = playbackState().name.lowercase(), segmentsUrl = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/segments", stateUrl = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/state", - blockedBy = if (window.isReady) null else SabrPlaybackDiagnostics.blocker(this) ?: window.blockedBy ?: "window pending", + blockedBy = if (ready) null else SabrPlaybackDiagnostics.blocker(this) ?: window.blockedBy ?: "window pending", playerTimeMs = request.playerTimeMs.coerceAtLeast(0L), readerHeadMs = readerHeadMs(), readerTailMs = readerTailMs(), diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt new file mode 100644 index 00000000..cc14f9e0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.findCachedMediaAt +import dev.typetype.server.services.playbackSegmentDurationMs +import dev.typetype.server.services.playbackSegmentStartMs +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +internal data class SabrProgressiveWindowSegment( + val sequence: Int, + val response: SabrPlaybackWindowSegment, +) + +internal data class SabrProgressiveWindowAppend(val nextSequence: Int, val coveredEndMs: Long) + +internal fun MutableList.appendProgressiveWindowSegment( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + predictedSequence: Int, + targetMs: Long, +): SabrProgressiveWindowAppend? { + val segment = holder.progressiveWindowSegment(format, predictedSequence, targetMs) ?: return null + if (segment.sequence != predictedSequence) holder.session.streamState.jumpBufferedTo(format, segment.sequence) + add(segment.response) + return SabrProgressiveWindowAppend( + segment.sequence + 1, + segment.response.startMs + segment.response.durationMs, + ) +} + +internal fun SabrSessionHolder.progressiveWindowSegment( + format: YoutubeSabrFormat, + predictedSequence: Int, + targetMs: Long, +): SabrProgressiveWindowSegment? { + val request = SabrSegmentRequest.media(format, predictedSequence) + val exact = session.getReadableSegment(request)?.takeIf { it.covers(targetMs, this, format) } + val actual = exact ?: session.findCachedMediaAt(format, targetMs, predictedSequence) + actual?.let(::observeMediaSegment) + val sequence = actual?.header?.sequenceNumber ?: predictedSequence + val startMs = actual?.header?.startMs?.takeIf { it >= 0L } + ?: playbackSegmentStartMs(format, sequence) + val durationMs = actual?.header?.durationMs?.takeIf { it > 0L } + ?: playbackSegmentDurationMs(format, sequence) + if (startMs < 0L || durationMs <= 0L) return null + return SabrProgressiveWindowSegment( + sequence, + SabrPlaybackWindowSegment( + url = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/${format.itag}/segment/$sequence?generation=${activeGeneration()}", + startMs = startMs, + durationMs = durationMs, + ), + ) +} + +private fun SabrMediaSegment.covers( + targetMs: Long, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, +): Boolean { + val startMs = header.startMs.takeIf { it >= 0L } + ?: holder.playbackSegmentStartMs(format, header.sequenceNumber) + val durationMs = header.durationMs.takeIf { it > 0L } + ?: holder.playbackSegmentDurationMs(format, header.sequenceNumber) + return targetMs >= startMs - TIMING_TOLERANCE_MS && targetMs < startMs + durationMs +} + +private const val TIMING_TOLERANCE_MS = 2L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt new file mode 100644 index 00000000..4ba20e8e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withTimeoutOrNull +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest + +internal class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { + suspend fun fetch( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult = if (holder.livePlaybackSnapshot()?.active == true) { + fetchLive(holder, request, timeoutMs, generation) + } else { + fetchProgressive(holder, request, timeoutMs, generation) + } + + private suspend fun fetchProgressive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult { + holder.session.getReadableSegment(request)?.let { + return stream(holder, request, it, generation) + } + sessionStore.requestSegmentDemand(holder, request, generation) + val segment = runInterruptible(Dispatchers.IO) { + holder.session.awaitReadableSegment(request, timeoutMs) + } + return if (segment == null) { + SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) + } else { + stream(holder, request, segment, generation) + } + } + + private suspend fun fetchLive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult { + sessionStore.cachedSegment(holder, request)?.let { + holder.markServed(it, generation) + return SabrPlaybackSegmentResult.Ready(it.mimeType, it.bytes) + } + sessionStore.requestSegmentDemand(holder, request, generation) + val segment = awaitCachedLive(holder, request, timeoutMs) + return if (segment == null) SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) else { + holder.clearSegmentDemand(request) + holder.markServed(segment, generation) + SabrPlaybackSegmentResult.Ready(segment.mimeType, segment.bytes) + } + } + + private suspend fun awaitCachedLive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + ): CachedSabrSegment? = withTimeoutOrNull(timeoutMs) { + var segment = sessionStore.cachedSegment(holder, request) + while (segment == null && holder.terminalFailure() == null && holder.networkFailure() == null) { + delay(SEGMENT_WAIT_MS) + segment = sessionStore.cachedSegment(holder, request) + if (segment == null) segment = followingLiveSegment(holder, request) + } + segment + } + + private suspend fun followingLiveSegment( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + ): CachedSabrSegment? { + val targetMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) + return sessionStore.findCachedPlaybackMediaAt( + holder = holder, + format = request.format, + targetMs = targetMs, + predictedSequence = request.sequenceNumber, + allowFollowing = true, + )?.takeUnless { + holder.failLivePlaybackDiscontinuity( + request.format, + targetMs, + it, + holder.lastServedSequence(request.format) != null, + ) + } + } + + private fun stream( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + segment: SabrMediaSegment, + generation: Long, + ): SabrPlaybackSegmentResult.Stream { + holder.clearSegmentDemand(request) + return SabrPlaybackSegmentResult.Stream(request.format.mimeType.orEmpty(), segment, holder, generation) + } + + private companion object { + const val REPOSITIONING = "repositioning" + const val SEGMENT_WAIT_MS = 250L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt index e3e123eb..62774b80 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt @@ -1,7 +1,15 @@ package dev.typetype.server.services +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment + internal sealed class SabrPlaybackSegmentResult { data class Ready(val mimeType: String, val bytes: ByteArray) : SabrPlaybackSegmentResult() + data class Stream( + val mimeType: String, + val segment: SabrMediaSegment, + val holder: SabrSessionHolder, + val generation: Long, + ) : SabrPlaybackSegmentResult() data class Retry(val holder: SabrSessionHolder, val status: String) : SabrPlaybackSegmentResult() data class Stale(val holder: SabrSessionHolder) : SabrPlaybackSegmentResult() data object InvalidSequence : SabrPlaybackSegmentResult() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index b3fb8190..6eb0d40d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services -import kotlinx.coroutines.delay import kotlinx.coroutines.withTimeoutOrNull import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { + private val mediaFetcher = SabrPlaybackMediaFetcher(sessionStore) suspend fun prepare( videoId: String, userId: String, @@ -145,48 +145,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS if (generation > activeGeneration) return SabrPlaybackSegmentResult.InvalidGeneration val request = SabrSegmentRequest.media(format, sequence) if (generation < activeGeneration) return staleMedia(holder, request) - sessionStore.cachedSegment(holder, request)?.let { - holder.markServed(it, generation) - return SabrPlaybackSegmentResult.Ready(it.mimeType, it.bytes) - } - sessionStore.requestSegmentDemand(holder, request, generation) - val segment = awaitCachedSegment(holder, request, timeoutMs) - return if (segment == null) SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) else { - holder.clearSegmentDemand(request) - holder.markServed(segment, generation) - SabrPlaybackSegmentResult.Ready(segment.mimeType, segment.bytes) - } - } - - private suspend fun awaitCachedSegment( - holder: SabrSessionHolder, - request: SabrSegmentRequest, - timeoutMs: Long, - ): CachedSabrSegment? = withTimeoutOrNull(timeoutMs) { - var segment = sessionStore.cachedSegment(holder, request) - while (segment == null && holder.terminalFailure() == null && holder.networkFailure() == null) { - delay(SEGMENT_WAIT_MS) - segment = sessionStore.cachedSegment(holder, request) - if (segment == null && holder.livePlaybackSnapshot()?.active == true) { - val targetMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) - val following = sessionStore.findCachedPlaybackMediaAt( - holder = holder, - format = request.format, - targetMs = targetMs, - predictedSequence = request.sequenceNumber, - allowFollowing = true, - ) - segment = following?.takeUnless { - holder.failLivePlaybackDiscontinuity( - request.format, - targetMs, - it, - holder.lastServedSequence(request.format) != null, - ) - } - } - } - segment + return mediaFetcher.fetch(holder, request, timeoutMs, generation) } private suspend fun staleMedia(holder: SabrSessionHolder, request: SabrSegmentRequest): SabrPlaybackSegmentResult = @@ -199,9 +158,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS private companion object { const val PREPARING = "preparing" - const val REPOSITIONING = "repositioning" const val INITIALIZATION_PRELOAD_TIMEOUT_MS = 6_000L - const val SEGMENT_WAIT_MS = 250L const val LIVE_INITIAL_PUMPS = 8 const val OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS = 9_007_199_254_740_991L } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt index 8c6a9a03..db315d9d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt @@ -11,6 +11,9 @@ internal class SabrSegmentCache { fun put(holder: SabrSessionHolder, segment: SabrMediaSegment): Unit { val format = holder.formatForItag(segment.header.itag) ?: return holder.observeMediaSegment(segment) + if (holder.key.purpose == SabrSessionPurpose.PLAYBACK && + !holder.expectsLive() && !segment.header.isInitSegment + ) return val mimeType = format.mimeType.orEmpty() val mediaParts = segment.takeIf { holder.expectsLive() && !it.header.isInitSegment } ?.let { SabrLiveMediaNormalizer.split(mimeType, it.data) } diff --git a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt index 66e4e649..75475493 100644 --- a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt @@ -119,7 +119,7 @@ class SabrPlaybackGranularRoutesTest { } @Test - fun `window queues missing first video without direct fetch`() = testApplication { + fun `window advertises missing first video without direct fetch`() = testApplication { val store = audioOnlyStore() val holder = holder() every { store.lookupByToken("session-token") } returns holder @@ -130,13 +130,13 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody()) } - assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(response.bodyAsText().contains("video:136:1 pending")) + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().contains("segment/1")) verify(atLeast = 1) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } } @Test - fun `window queues video first when blocked tracks have equal coverage`() = testApplication { + fun `window keeps stale timeline pending and queues both tracks`() = testApplication { val store = emptyStore() val holder = holder() every { store.lookupByToken("session-token") } returns holder @@ -147,9 +147,7 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody(340_000L)) } - val body = response.bodyAsText() assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(body.contains("video:136:1 pending")) verify(exactly = 2) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } coVerify(exactly = 0) { store.fetchInitializationData(holder, any()) } } @@ -232,6 +230,7 @@ class SabrPlaybackGranularRoutesTest { val state = mockk() every { session.streamState } returns state every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null every { session.isBeyondEnd(any()) } returns false every { state.setActiveTrackTypes(true, true) } returns Unit every { state.setPlaybackRate(any()) } returns Unit diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt index af76683c..420f27b0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt @@ -7,7 +7,6 @@ import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertSame @@ -107,19 +106,19 @@ class SabrPlaybackSessionServiceTest { } @Test - fun `cached media segment after demand returns bytes`() = runTest { + fun `readable media segment after demand returns progressive stream`() = runTest { val audio = format(140, isAudio = true) val holder = holder(audio, format(137, isAudio = false)) - val segment = cachedSegment(140, 4, byteArrayOf(1, 2, 3)) + val segment = mockk() + every { holder.session.awaitReadableSegment(any(), any()) } returns segment val store = mockk() - coEvery { store.cachedSegment(holder, any()) } returnsMany listOf(null, segment) every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit val result = SabrPlaybackSessionService(store).fetchMedia(holder, audio, sequence = 4, timeoutMs = 50L, generation = 0L) - val ready = result as SabrPlaybackSegmentResult.Ready - assertEquals("audio/mp4", ready.mimeType) - assertArrayEquals(byteArrayOf(1, 2, 3), ready.bytes) + val stream = result as SabrPlaybackSegmentResult.Stream + assertEquals("audio/mp4", stream.mimeType) + assertSame(segment, stream.segment) verify(exactly = 1) { store.requestSegmentDemand(holder, any(), 0L) } } @@ -245,6 +244,8 @@ class SabrPlaybackSessionServiceTest { val state = mockk(relaxed = true) every { session.streamState } returns state every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null + every { session.awaitReadableSegment(any(), any()) } returns null every { session.isBeyondEnd(any()) } returns false every { session.prepareForInitialization(any()) } returns Unit every { state.setActiveTrackTypes(any(), any()) } returns Unit diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt index 2be01656..4552ef57 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt @@ -2,9 +2,10 @@ package dev.typetype.server.services import io.mockk.every import io.mockk.mockk +import io.mockk.verify import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment @@ -17,27 +18,19 @@ import java.time.Instant class SabrSegmentCacheTest { @Test - fun `cache stores and reads segment metadata and bytes`() { + fun `vod cache observes media without copying segment bytes`() { val segmentCache = SabrSegmentCache() val audio = format(140, isAudio = true) val video = format(137, isAudio = false) - val holder = holder(audio, video) + val holder = holder(audio, video, SabrSessionPurpose.PLAYBACK) val bytes = byteArrayOf(1, 2, 3) val segment = segment(itag = 140, sequence = 4, bytes = bytes) val request = SabrSegmentRequest.media(audio, 4) segmentCache.put(holder, segment) - val cached = segmentCache.get(holder, request) - - requireNotNull(cached) - assertEquals(140, cached.itag) - assertEquals(4, cached.sequence) - assertEquals(1234L, cached.startMs) - assertEquals(9985L, cached.durationMs) - assertEquals("audio/mp4", cached.mimeType) - assertArrayEquals(byteArrayOf(1, 2, 3), cached.bytes) - assertSame(bytes, cached.bytes) + assertNull(segmentCache.get(holder, request)) assertEquals(4, holder.observedMediaSegment(audio)?.header?.sequenceNumber) + verify(exactly = 0) { segment.data } } @Test @@ -56,7 +49,11 @@ class SabrSegmentCacheTest { assertArrayEquals(media, requireNotNull(segmentCache.get(holder, request)).bytes) } - private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): SabrSessionHolder { + private fun holder( + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + purpose: SabrSessionPurpose = SabrSessionPurpose.MANIFEST, + ): SabrSessionHolder { val session = mockk() val state = mockk() every { session.streamState } returns state @@ -67,7 +64,7 @@ class SabrSegmentCacheTest { audioFormat = audio, videoFormat = video, sessionToken = "token", - key = SabrSessionKey("video", "user", 140, null, 137, 0L), + key = SabrSessionKey("video", "user", 140, null, 137, 0L, purpose), lastRequestAt = Instant.EPOCH, ) } From a422b9d98761eb4e1e91b01f564fac5ac9a5afa9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 15:43:43 +0200 Subject: [PATCH 57/68] test: cover progressive playback windows --- .../SabrProgressivePlaybackWindowTest.kt | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt new file mode 100644 index 00000000..bb86d165 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionStore +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrProgressivePlaybackWindowTest { + @Test + fun `vod window exposes timeline segments before their payload completes`() = runTest { + val audio = format(140, isAudio = true) + val video = format(308, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 1 + every { state.getSegmentStartMs(any(), 1) } returns 0L + every { state.getSegmentEndMs(audio, 1) } returns 9_985L + every { state.getSegmentEndMs(video, 1) } returns 4_000L + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getReadableSegment(any()) } returns null + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest(0L, 0L, video.itag, audio.itag, bufferGoalMs = 2_500L), + ) + + assertTrue(result.isReady) + assertTrue(result.blockedRequests.isEmpty()) + assertEquals(4_000L, requireNotNull(result.response.video).segments.single().durationMs) + assertEquals(9_985L, result.response.audio.segments.single().durationMs) + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk().also { + every { it.itag } returns itag + every { it.isAudio } returns isAudio + every { it.mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { it.approxDurationMs } returns 900_000L + } +} From 8ff8fee64136f67f1da063dd51fe31878686eb6f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 18:00:53 +0200 Subject: [PATCH 58/68] test: verify readable progressive windows --- .../SabrProgressivePlaybackWindowTest.kt | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index bb86d165..1b693750 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -8,8 +8,11 @@ import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo @@ -28,8 +31,11 @@ class SabrProgressivePlaybackWindowTest { every { state.getSegmentEndMs(audio, 1) } returns 9_985L every { state.getSegmentEndMs(video, 1) } returns 4_000L val session = mockk(relaxed = true) + val audioSegment = readableSegment(audio, durationMs = 9_985L) + val videoSegment = readableSegment(video, durationMs = 4_000L) every { session.streamState } returns state - every { session.getReadableSegment(any()) } returns null + every { session.getReadableSegment(match { it.format == audio }) } returns audioSegment + every { session.getReadableSegment(match { it.format == video }) } returns videoSegment every { session.getCachedSegment(any()) } returns null val holder = holder(session, audio, video) val store = mockk() @@ -41,10 +47,25 @@ class SabrProgressivePlaybackWindowTest { assertTrue(result.isReady) assertTrue(result.blockedRequests.isEmpty()) + assertFalse(audioSegment.isComplete) + assertFalse(videoSegment.isComplete) assertEquals(4_000L, requireNotNull(result.response.video).segments.single().durationMs) assertEquals(9_985L, result.response.audio.segments.single().durationMs) } + private fun readableSegment(format: YoutubeSabrFormat, durationMs: Long): SabrMediaSegment { + val header = mockk() + every { header.itag } returns format.itag + every { header.isInitSegment } returns false + every { header.sequenceNumber } returns 1 + every { header.startMs } returns 0L + every { header.durationMs } returns durationMs + return mockk().also { + every { it.header } returns header + every { it.isComplete } returns false + } + } + private fun holder( session: YoutubeSabrSession, audio: YoutubeSabrFormat, From 09db8fd3776f6f460e31c9e843caa61bda5bc4bf Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 18:22:43 +0200 Subject: [PATCH 59/68] fix: align VOD audio windows to the playhead --- .../routes/SabrPlaybackWindowBuilder.kt | 4 +- .../routes/SabrPlaybackWindowBuilderTest.kt | 2 +- .../SabrProgressivePlaybackWindowTest.kt | 39 +++++++++++++++++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt index a67ee33a..58f28b67 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -28,11 +28,12 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi if (effectiveRequest.audioOnly) return buildAudioOnly(holder, effectiveRequest, live?.toResponse()) val video = buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, live?.active == true) val decodeStartMs = video.track.segments.firstOrNull()?.startMs ?: effectiveRequest.playerTimeMs + val audioStartMs = if (live?.active == true) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs val audio = buildTrack( holder, holder.audioFormat, effectiveRequest, - minOf(effectiveRequest.playerTimeMs, decodeStartMs), + audioStartMs, live?.active == true, ) val playbackStartMs = resolvedPlaybackStartMs( @@ -191,7 +192,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi atEnd = atEnd, ) } - private fun CachedSabrSegment.toWindowSegment( holder: SabrSessionHolder, format: YoutubeSabrFormat, diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt index e813a8a5..17aeaeac 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt @@ -31,7 +31,7 @@ class SabrPlaybackWindowBuilderTest { val streamState = mockk(relaxed = true) every { session.streamState } returns streamState every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 491_203L) } returns 98 - every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 488_200L) } returns 49 + every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 491_203L) } returns 49 every { session.getCachedSegment(any()) } answers { firstArg().takeIf { it.format.itag == 299 && it.sequenceNumber == 101 } ?.let { mediaSegment(sequence = 101, startMs = 488_200L, durationMs = 6_500L) } diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index 1b693750..031d11ce 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -21,6 +21,34 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState import java.time.Instant class SabrProgressivePlaybackWindowTest { + @Test + fun `vod audio starts at the playhead rather than the preceding video keyframe`() = runTest { + val audio = format(140, isAudio = true) + val video = format(137, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(video, 100_000L) } returns 16 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 100_000L) } returns 11 + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getReadableSegment(match { it.format == video && it.sequenceNumber == 16 }) } returns + readableSegment(video, sequence = 16, startMs = 95_000L, durationMs = 7_000L) + every { session.getReadableSegment(match { it.format == audio && it.sequenceNumber == 11 }) } returns + readableSegment(audio, sequence = 11, startMs = 99_845L, durationMs = 9_985L) + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest(0L, 100_000L, video.itag, audio.itag, bufferGoalMs = 1_000L), + ) + + assertTrue(result.isReady) + assertEquals(95_000L, requireNotNull(result.response.video).segments.single().startMs) + assertEquals(99_845L, result.response.audio.segments.single().startMs) + } + @Test fun `vod window exposes timeline segments before their payload completes`() = runTest { val audio = format(140, isAudio = true) @@ -53,12 +81,17 @@ class SabrProgressivePlaybackWindowTest { assertEquals(9_985L, result.response.audio.segments.single().durationMs) } - private fun readableSegment(format: YoutubeSabrFormat, durationMs: Long): SabrMediaSegment { + private fun readableSegment( + format: YoutubeSabrFormat, + sequence: Int = 1, + startMs: Long = 0L, + durationMs: Long, + ): SabrMediaSegment { val header = mockk() every { header.itag } returns format.itag every { header.isInitSegment } returns false - every { header.sequenceNumber } returns 1 - every { header.startMs } returns 0L + every { header.sequenceNumber } returns sequence + every { header.startMs } returns startMs every { header.durationMs } returns durationMs return mockk().also { every { it.header } returns header From 235df60dacfde3a07375caa8a2d3db7bcb25a551 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 20:33:20 +0200 Subject: [PATCH 60/68] perf: reuse SABR bootstrap preparation --- .../server/ExtractionServiceRegistry.kt | 6 ++--- .../services/SabrBootstrapStreamService.kt | 21 ++++++++--------- .../server/services/SabrInfoFetcher.kt | 17 ++++++-------- .../server/services/SabrSessionStore.kt | 2 +- .../server/services/TokenYoutubeSession.kt | 6 +++++ .../SabrBootstrapStreamServiceTest.kt | 23 +++++++++++++++---- 6 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 5791d986..2dc6fd25 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -134,10 +134,8 @@ internal class ExtractionServiceRegistry( ), YouTubeSubtitleCache(cache), ) - val youtubeSabrBootstrapStreamService = CachedStreamService( - YoutubeScopedStreamService(SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient)), - cache, - "stream-youtube-sabr-bootstrap:v1", + val youtubeSabrBootstrapStreamService = YoutubeScopedStreamService( + SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient), ) val nicoNicoStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-niconico:v1") val bilibiliStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-bilibili:v1") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt index a1998fb9..771aed8c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -2,24 +2,21 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope internal class SabrBootstrapStreamService( private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, ) : StreamService { - override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { + override suspend fun getStreamInfo(url: String): ExtractionResult { val videoId = youtubeVideoId(url) - ?: return@coroutineScope ExtractionResult.BadRequest("Invalid YouTube URL") - val prepared = async { sessionStore.fetchInfo(videoId, cachedFirst = true) } - val session = async { tokenSessionClient.fetchPlaybackSession(videoId) } - val metadata = session.await() - ?: return@coroutineScope ExtractionResult.Failure("SABR bootstrap metadata unavailable") - if (prepared.await() == null) { - return@coroutineScope ExtractionResult.Failure("SABR playback formats unavailable") - } - ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) + ?: return ExtractionResult.BadRequest("Invalid YouTube URL") + val metadata = tokenSessionClient.fetchPlaybackSession(videoId) + ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") + val prepared = metadata.preparedSabrInfo() + ?: sessionStore.fetchInfo(videoId, cachedFirst = true) + ?: return ExtractionResult.Failure("SABR playback formats unavailable") + sessionStore.rememberPreparedInfo(videoId, prepared) + return ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index 6e5e5e74..e71c0bb5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -59,9 +59,12 @@ internal class SabrInfoFetcher( suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit { repository.rememberInitialization(videoId, info) - fetchInfoOnce(videoId, startTimeMs = 0L) - ?.takeIf { it.hasAudioAndVideoFormats() } - ?.let { repository.putPrepared(videoId, startTimeMs = 0L, it) } + fetchInfo(videoId, startTimeMs = 0L, cachedFirst = true) + } + + suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit { + repository.rememberInitialization(videoId, prepared.info) + repository.putPrepared(videoId, startTimeMs = 0L, prepared) } suspend fun invalidatePlayback(videoId: String): Unit = repository.invalidatePlayback(videoId) @@ -87,13 +90,7 @@ internal class SabrInfoFetcher( ): SabrPreparedInfo? = withTimeoutOrNull(SabrSessionStoreDefaults.INFO_TIMEOUT_MS) { val tokenSession = sessionClient?.fetchPlaybackSession(videoId, isolatedPlayback) - tokenSession?.token - ?.takeIf { it.visitorData == tokenSession.info.visitorData } - ?.let { sessionToken -> - SabrPreparedInfo(tokenSession.info, sessionToken, tokenSession.isLive, tokenSession.isLiveContent) - .takeIf { it.hasAudioAndVideoFormats() } - ?.let { return@withTimeoutOrNull it } - } + tokenSession?.preparedSabrInfo()?.let { return@withTimeoutOrNull it } val token = tokenClient.fetch(videoId) ?: return@withTimeoutOrNull null.also { logger.warn( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 5f666549..57f705e3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -151,7 +151,7 @@ internal class SabrSessionStore( internal suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) - + internal suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) internal suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) internal suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = diff --git a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt index dcfbb3d7..283479f6 100644 --- a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt +++ b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt @@ -18,3 +18,9 @@ internal data class TokenYoutubeSession( val isLiveContent: Boolean, val hlsUrl: String = "", ) + +internal fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { + val boundToken = token?.takeIf { it.visitorData == info.visitorData } ?: return null + return SabrPreparedInfo(info, boundToken, isLive, isLiveContent) + .takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt index bbcf63f7..9be05aad 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt @@ -17,8 +17,9 @@ class SabrBootstrapStreamServiceTest { val sessionStore = mockk() val tokenClient = mockk() val prepared = preparedInfo() - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns prepared - coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession(prepared.info) + val session = tokenSession(prepared.info, tokenBundle()) + coEvery { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } returns Unit + coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns session val service = SabrBootstrapStreamService(sessionStore, tokenClient) val result = service.getStreamInfo(YOUTUBE_URL) @@ -27,7 +28,8 @@ class SabrBootstrapStreamServiceTest { assertEquals("Bootstrap title", response.title) assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) assertEquals(listOf(140), response.audioStreams.map { it.itag }) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { sessionStore.fetchInfo(any(), any()) } + coVerify(exactly = 1) { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } coVerify(exactly = 1) { tokenClient.fetchPlaybackSession(VIDEO_ID) } } @@ -58,12 +60,22 @@ class SabrBootstrapStreamServiceTest { every { audio.audioTrackId } returns "en.4" val info = mockk() every { info.formats } returns listOf(video, audio) + every { info.visitorData } returns VISITOR_DATA return SabrPreparedInfo(info, null) } - private fun tokenSession(info: YoutubeSabrInfo): TokenYoutubeSession = TokenYoutubeSession( + private fun tokenBundle() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "player-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = VISITOR_DATA, + videoBoundPoToken = "media-token", + videoBoundPoTokenBytes = byteArrayOf(2), + ) + + private fun tokenSession(info: YoutubeSabrInfo, token: SabrTokenBundle? = null): TokenYoutubeSession = TokenYoutubeSession( info = info, - token = null, + token = token, title = "Bootstrap title", author = "Bootstrap channel", channelId = "channel-id", @@ -80,5 +92,6 @@ class SabrBootstrapStreamServiceTest { private companion object { const val VIDEO_ID = "f6f3PhauXyg" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val VISITOR_DATA = "visitor-data" } } From ed9649f0903158f6573fe3c83a9b20dc24c1d405 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 25 Aug 2026 21:10:16 +0200 Subject: [PATCH 61/68] perf: preload SABR initialization concurrently --- .../SabrPlaybackInitializationPreloader.kt | 14 +++++++++----- .../SabrPlaybackInitializationFailureTest.kt | 13 +++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt index b343d061..10064135 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt @@ -1,5 +1,7 @@ package dev.typetype.server.services +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withTimeoutOrNull internal data class SabrPlaybackInitializationPreloadResult( @@ -23,10 +25,12 @@ internal object SabrPlaybackInitializationPreloader { audioOnly: Boolean, timeoutMs: Long, ): SabrPlaybackInitializationPreloadResult = withTimeoutOrNull(timeoutMs) { - val video = holder.videoFormat - .takeUnless { audioOnly } - ?.let { sessionStore.fetchInitializationData(holder, it) } - val audio = sessionStore.fetchInitializationData(holder, holder.audioFormat) - SabrPlaybackInitializationPreloadResult(video, audio) + coroutineScope { + val video = holder.videoFormat + .takeUnless { audioOnly } + ?.let { format -> async { sessionStore.fetchInitializationData(holder, format) } } + val audio = async { sessionStore.fetchInitializationData(holder, holder.audioFormat) } + SabrPlaybackInitializationPreloadResult(video?.await(), audio.await()) + } } ?: SabrPlaybackInitializationPreloadResult(null, null) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt index 8bf94800..59b34411 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt @@ -3,7 +3,6 @@ package dev.typetype.server.services import dev.typetype.server.routes.SabrPlaybackRecovery import io.mockk.coEvery import io.mockk.coVerify -import io.mockk.coVerifyOrder import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -49,6 +48,10 @@ class SabrPlaybackInitializationFailureTest { delay(Long.MAX_VALUE) null } + coEvery { store.fetchInitializationData(holder, audio) } coAnswers { + delay(Long.MAX_VALUE) + null + } coEvery { store.invalidatePlaybackInfo("video") } returns Unit val result = SabrPlaybackSessionService(store).prepare( @@ -65,7 +68,7 @@ class SabrPlaybackInitializationFailureTest { assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) verify(exactly = 0) { store.startPump(any()) } - coVerify(exactly = 0) { store.fetchInitializationData(holder, audio) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } @Test @@ -113,10 +116,8 @@ class SabrPlaybackInitializationFailureTest { assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) verify(exactly = 0) { store.startPump(any()) } coVerify(exactly = 1) { store.invalidatePlaybackInfo("video") } - coVerifyOrder { - store.fetchInitializationData(holder, video) - store.fetchInitializationData(holder, audio) - } + coVerify(exactly = 1) { store.fetchInitializationData(holder, video) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } @Test From 2e93c961c481f2a830441a8af1c9449a34d8ae77 Mon Sep 17 00:00:00 2001 From: User Date: Tue, 25 Aug 2026 16:39:20 -0700 Subject: [PATCH 62/68] feat: expose subscription group memberships Add an account-scoped read model for subscription channels and their complete group assignments so the frontend can render and edit memberships without issuing one filtered subscription request per group. Constraint: Keep the shared SubscriptionItem unchanged because feeds, backups, imports, RSS, and recommendations reuse it Rejected: Add groupIds to SubscriptionItem | would query and serialize group data in unrelated paths Rejected: Fetch each group projection from the frontend | creates N requests for N groups Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep groupIds account-scoped and include empty arrays for ungrouped subscriptions Tested: JDK 25 clean check, 1,132 tests, shadowJar, OpenAPI validation, and live HTTP QA Not-tested: Frontend integration is intentionally deferred --- openapi.yaml | 2 + openapi/components/subscriptions.yaml | 12 +++++ openapi/paths/subscriptions.yaml | 14 ++++++ .../models/SubscriptionGroupMembershipItem.kt | 12 +++++ .../server/routes/SubscriptionsRoutes.kt | 5 ++ .../server/services/SubscriptionsService.kt | 43 ++++++++++++++-- .../server/SubscriptionGroupsRoutesTest.kt | 50 +++++++++++++++++++ 7 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt diff --git a/openapi.yaml b/openapi.yaml index 849c1c99..ba7e65b9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -45,6 +45,7 @@ paths: /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } + /subscriptions/group-memberships: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupMemberships } /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } @@ -160,6 +161,7 @@ components: SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } SubscriptionItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionItem } + SubscriptionGroupMembershipItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipItem } SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index 5894cc5f..c811f8a3 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -6,6 +6,18 @@ SubscriptionItem: name: { type: string } avatarUrl: { type: string } subscribedAt: { type: integer, format: int64 } +SubscriptionGroupMembershipItem: + type: object + required: [channelUrl, name, avatarUrl, subscribedAt, groupIds] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } + subscribedAt: { type: integer, format: int64 } + groupIds: + type: array + uniqueItems: true + items: { type: string, format: uuid } SubscriptionCreateRequest: type: object required: [channelUrl, name, avatarUrl] diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index 9f9b4863..ac2dfa16 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -52,6 +52,20 @@ Subscriptions: '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroupMemberships: + get: + tags: [user-data] + summary: List subscriptions with their group memberships + description: Returns every current subscription once, with all account-owned group IDs assigned to that channel. Ungrouped subscriptions have an empty groupIds array. + responses: + '200': + description: Account-scoped subscriptions and their complete group memberships. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipItem } + '401': { $ref: ../components/common.yaml#/JsonError } SubscriptionGroups: get: tags: [user-data] diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt new file mode 100644 index 00000000..b4a6fa60 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupMembershipItem( + val channelUrl: String, + val name: String, + val avatarUrl: String, + val subscribedAt: Long, + val groupIds: List, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index becd5d9b..33aac871 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -27,6 +27,11 @@ fun Route.subscriptionsRoutes( warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), ) { + get("/subscriptions/group-memberships") { + call.withJwtAuth(authService) { userId -> + call.respond(subscriptionsService.getAllWithGroupMemberships(userId)) + } + } get("/subscriptions") { call.withJwtAuth(authService) { userId -> val parsed = call.parseSubscriptionSelection() diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 6e90ab8f..70da17df 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -2,7 +2,9 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupMembershipItem import dev.typetype.server.models.SubscriptionItem import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder @@ -19,14 +21,37 @@ class SubscriptionsService { selection: SubscriptionSelection = SubscriptionSelection.All, ): List = DatabaseFactory.query { val selectedUrls = selectedChannelUrls(userId, selection) - val items = SubscriptionsTable.selectAll() - .where { SubscriptionsTable.userId eq userId } - .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) - .map { it.toItem() } + subscriptionItems(userId) .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } - SubscriptionAvatarRepairer.repair(userId = userId, items = items) } + suspend fun getAllWithGroupMemberships(userId: String): List = + DatabaseFactory.query { + val groupIdsByChannel = SubscriptionGroupMembershipsTable + .innerJoin(SubscriptionGroupsTable) + .selectAll() + .where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupsTable.userId eq userId) + } + .groupBy( + keySelector = { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionGroupMembershipsTable.channelUrl]) + }, + valueTransform = { it[SubscriptionGroupMembershipsTable.groupId] }, + ) + .mapValues { (_, groupIds) -> groupIds.sorted() } + subscriptionItems(userId).map { item -> + SubscriptionGroupMembershipItem( + channelUrl = item.channelUrl, + name = item.name, + avatarUrl = item.avatarUrl, + subscribedAt = item.subscribedAt, + groupIds = groupIdsByChannel[item.channelUrl].orEmpty(), + ) + } + } + suspend fun getChannelUrls(userId: String, selection: SubscriptionSelection): Set = DatabaseFactory.query { selectedChannelUrls(userId, selection) } @@ -55,6 +80,14 @@ class SubscriptionsService { SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0 } + private fun subscriptionItems(userId: String): List { + val items = SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq userId } + .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) + .map { it.toItem() } + return SubscriptionAvatarRepairer.repair(userId = userId, items = items) + } + private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { val all = SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt index 70dd04b3..55c0fa76 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -6,6 +6,7 @@ import dev.typetype.server.routes.subscriptionGroupsRoutes import dev.typetype.server.routes.subscriptionsRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.delete import io.ktor.client.request.get @@ -25,6 +26,9 @@ import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -37,6 +41,8 @@ class SubscriptionGroupsRoutesTest { private val auth = AuthService.fixed(TEST_USER_ID) companion object { + private const val FOREIGN_USER_ID = "foreign-user" + @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() @@ -61,6 +67,11 @@ class SubscriptionGroupsRoutesTest { assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/groups").status) } + @Test + fun `group membership projection requires authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/group-memberships").status) + } + @Test fun `groups can be created listed renamed and deleted`() = withApp { val create = client.post("/subscriptions/groups") { @@ -121,6 +132,45 @@ class SubscriptionGroupsRoutesTest { assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) } + @Test + fun `group membership projection returns account scoped memberships with subscription data`() = withApp { + val sharedChannel = channel("shared") + subscriptions.add(TEST_USER_ID, SubscriptionItem(sharedChannel, "Shared", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("ungrouped"), "Ungrouped", "")) + subscriptions.add(FOREIGN_USER_ID, SubscriptionItem(sharedChannel, "Foreign shared", "")) + + val ownGroups = listOf(createGroup("Own"), createGroup("Another")) + ownGroups.forEach { group -> + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"$sharedChannel"}""") + }.status) + } + val foreignGroup = requireNotNull( + (groups.create(FOREIGN_USER_ID, "Foreign") as? SubscriptionGroupWriteResult.Success)?.group, + ) + groups.addSubscription(FOREIGN_USER_ID, foreignGroup.id, sharedChannel) + + val response = authorizedGet("/subscriptions/group-memberships") + assertEquals(HttpStatusCode.OK, response.status) + val items = Json.parseToJsonElement(response.bodyAsText()).jsonArray.map { it.jsonObject } + assertEquals(2, items.size) + + val shared = items.single { it.getValue("channelUrl").jsonPrimitive.content == sharedChannel } + assertEquals("Shared", shared.getValue("name").jsonPrimitive.content) + assertEquals("avatar", shared.getValue("avatarUrl").jsonPrimitive.content) + assertTrue(shared.getValue("subscribedAt").jsonPrimitive.content.toLong() > 0) + assertEquals( + ownGroups.map { it.id }.sorted(), + shared.getValue("groupIds").jsonArray.map { it.jsonPrimitive.content }, + ) + + val ungrouped = items.single { + it.getValue("channelUrl").jsonPrimitive.content == channel("ungrouped") + } + assertEquals(emptyList(), ungrouped.getValue("groupIds").jsonArray.map { it.jsonPrimitive.content }) + } + @Test fun `invalid or inaccessible filters fail explicitly`() = withApp { assertEquals(HttpStatusCode.BadRequest, authorizedGet("/subscriptions") { From a3a1530a92075ca8ad377fc2fad1da6ed58800ef Mon Sep 17 00:00:00 2001 From: User Date: Tue, 25 Aug 2026 17:16:34 -0700 Subject: [PATCH 63/68] feat: support bulk subscription group membership updates Let clients add or remove many subscribed channels from one group in a single account-scoped transaction while preserving the shipped singular request forms. Constraint: The singular membership contract shipped in v1.6.0 and must remain compatible Rejected: Cross-group membership delta endpoint | bulk organization is naturally scoped to one group Rejected: Repeated delete query parameters | encoded channel URLs can exceed practical URL limits Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep batch writes bounded, atomic, idempotent, and protected by SubscriptionMutationLock Tested: JDK 25 clean check, 1,134 tests, coverage, shadowJar, OpenAPI validation, focused retry regression, and live HTTP QA Not-tested: Frontend and Android integration are intentionally deferred --- openapi.yaml | 1 + openapi/components/subscriptions.yaml | 9 ++++ openapi/paths/subscriptions.yaml | 21 ++++++-- .../SubscriptionGroupMembershipRequest.kt | 5 +- .../server/routes/SubscriptionGroupsRoutes.kt | 50 ++++++++++++++---- .../services/SubscriptionGroupsService.kt | 51 +++++++++++++++---- .../server/SubscriptionGroupsRoutesTest.kt | 43 ++++++++++++++++ 7 files changed, 156 insertions(+), 24 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index ba7e65b9..63aec2da 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -165,6 +165,7 @@ components: SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + SubscriptionGroupMembershipBatchRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index c811f8a3..20a7476c 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -44,6 +44,15 @@ SubscriptionGroupMembershipRequest: required: [channelUrl] properties: channelUrl: { type: string, minLength: 1 } +SubscriptionGroupMembershipBatchRequest: + type: object + required: [channelUrls] + properties: + channelUrls: + type: array + minItems: 1 + maxItems: 500 + items: { type: string, minLength: 1 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index ac2dfa16..7783c19c 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -131,12 +131,16 @@ SubscriptionGroupChannels: schema: { type: string, format: uuid } put: tags: [user-data] - summary: Add a subscribed channel to a group + summary: Add subscribed channels to a group + description: Accepts the original singular channelUrl request or up to 500 channelUrls. Batch additions are atomic, canonicalized and deduplicated. requestBody: required: true content: application/json: - schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } responses: '204': { description: Membership exists. } '400': { $ref: ../components/common.yaml#/JsonError } @@ -144,12 +148,21 @@ SubscriptionGroupChannels: '404': { $ref: ../components/common.yaml#/JsonError } delete: tags: [user-data] - summary: Remove a subscribed channel from a group + summary: Remove subscribed channels from a group + description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. parameters: - name: url in: query - required: true + required: false schema: { type: string, minLength: 1 } + requestBody: + required: false + content: + application/json: + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } responses: '204': { description: Membership deleted. } '400': { $ref: ../components/common.yaml#/JsonError } diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt index 9a2fcb5f..d6befc62 100644 --- a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt @@ -3,4 +3,7 @@ package dev.typetype.server.models import kotlinx.serialization.Serializable @Serializable -data class SubscriptionGroupMembershipRequest(val channelUrl: String) +data class SubscriptionGroupMembershipRequest( + val channelUrl: String? = null, + val channelUrls: List? = null, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt index c80acb3d..be9f93c5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -45,21 +45,33 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut put("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - val request = runCatching { call.receive() }.getOrElse { - return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + when (val request = call.receiveMembershipChannels() ?: return@withJwtAuth) { + is MembershipChannels.Single -> call.respondMembership( + groupsService.addSubscription(userId, groupId, request.channelUrl), + ) + is MembershipChannels.Batch -> call.respondMembership( + groupsService.addSubscriptions(userId, groupId, request.channelUrls), + ) } - if (request.channelUrl.isBlank()) { - return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("channelUrl must not be blank")) - } - call.respondMembership(groupsService.addSubscription(userId, groupId, request.channelUrl)) } } delete("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - val channelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) - ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondMembership(groupsService.removeSubscription(userId, groupId, channelUrl)) + val queryChannelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) + if (queryChannelUrl != null) { + return@withJwtAuth call.respondMembership( + groupsService.removeSubscription(userId, groupId, queryChannelUrl), + ) + } + when (val request = call.receiveMembershipChannels() ?: return@withJwtAuth) { + is MembershipChannels.Single -> call.respondMembership( + groupsService.removeSubscription(userId, groupId, request.channelUrl), + ) + is MembershipChannels.Batch -> call.respondMembership( + groupsService.removeSubscriptions(userId, groupId, request.channelUrls), + ) + } } } } @@ -72,6 +84,26 @@ private suspend fun ApplicationCall.receiveGroupRequest(): SubscriptionGroupRequ null } +private sealed interface MembershipChannels { + data class Single(val channelUrl: String) : MembershipChannels + data class Batch(val channelUrls: List) : MembershipChannels +} + +private suspend fun ApplicationCall.receiveMembershipChannels(): MembershipChannels? { + val request = runCatching { receive() }.getOrNull() + val channelUrl = request?.channelUrl?.takeIf(String::isNotBlank) + val channelUrls = request?.channelUrls + val parsed = when { + channelUrl != null && channelUrls == null -> MembershipChannels.Single(channelUrl) + request?.channelUrl == null && channelUrls != null && + channelUrls.size in 1..SubscriptionGroupsService.MAX_MEMBERSHIP_CHANNELS && + channelUrls.all(String::isNotBlank) -> MembershipChannels.Batch(channelUrls) + else -> null + } + if (parsed == null) respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + return parsed +} + private suspend fun ApplicationCall.respondGroupWrite(result: SubscriptionGroupWriteResult, created: Boolean) { when (result) { is SubscriptionGroupWriteResult.Success -> if (created) respond(HttpStatusCode.Created, result.group) else { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 9529ad45..5c92769b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -9,6 +9,8 @@ import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.batchInsert import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll @@ -103,19 +105,31 @@ class SubscriptionGroupsService { userId: String, groupId: String, rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = addSubscriptions(userId, groupId, listOf(rawChannelUrl)) + + suspend fun addSubscriptions( + userId: String, + groupId: String, + rawChannelUrls: List, ): SubscriptionGroupMembershipResult = DatabaseFactory.query { SubscriptionMutationLock.acquire(userId) if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound - val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) - val subscriptionExists = SubscriptionsTable.selectAll().where { - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) - }.any() - if (!subscriptionExists) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound - SubscriptionGroupMembershipsTable.insertIgnore { - it[SubscriptionGroupMembershipsTable.groupId] = groupId - it[SubscriptionGroupMembershipsTable.userId] = userId - it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl - it[addedAt] = System.currentTimeMillis() + val channelUrls = rawChannelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl inList channelUrls) + }.mapTo(hashSetOf()) { it[SubscriptionsTable.channelUrl] } + if (subscribed.size != channelUrls.size) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound + val existing = SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList channelUrls) + }.mapTo(hashSetOf()) { it[SubscriptionGroupMembershipsTable.channelUrl] } + val addedAt = System.currentTimeMillis() + SubscriptionGroupMembershipsTable.batchInsert(channelUrls - existing, shouldReturnGeneratedValues = false) { url -> + this[SubscriptionGroupMembershipsTable.groupId] = groupId + this[SubscriptionGroupMembershipsTable.userId] = userId + this[SubscriptionGroupMembershipsTable.channelUrl] = url + this[SubscriptionGroupMembershipsTable.addedAt] = addedAt } SubscriptionGroupMembershipResult.Success } @@ -138,6 +152,22 @@ class SubscriptionGroupsService { } } + suspend fun removeSubscriptions( + userId: String, + groupId: String, + rawChannelUrls: List, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrls = rawChannelUrls.mapTo(hashSetOf(), ChannelUrlCanonicalizer::canonicalize) + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList channelUrls) + } + SubscriptionGroupMembershipResult.Success + } + suspend fun getChannelUrls(userId: String, groupId: String): List = DatabaseFactory.query { SubscriptionGroupMembershipsTable.selectAll().where { (SubscriptionGroupMembershipsTable.groupId eq groupId) and @@ -182,6 +212,7 @@ class SubscriptionGroupsService { companion object { const val MAX_GROUP_NAME_LENGTH = 100 + const val MAX_MEMBERSHIP_CHANNELS = 500 private const val UNIQUE_VIOLATION_SQL_STATE = "23505" } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt index 55c0fa76..46ebea82 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -132,6 +132,49 @@ class SubscriptionGroupsRoutesTest { assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) } + @Test + fun `membership routes add and remove multiple channels atomically`() = withApp { + val first = channel("one") + val second = channel("two") + val neverAdded = channel("three") + listOf(first, second, neverAdded).forEach { url -> + subscriptions.add(TEST_USER_ID, SubscriptionItem(url, url.substringAfterLast('/'), "")) + } + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second","$first"]}""") + }.status) + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second"]}""") + }.status) + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText() + assertTrue(grouped.contains(first)) + assertTrue(grouped.contains(second)) + assertTrue(!grouped.contains(neverAdded)) + + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second","$neverAdded"]}""") + }.status) + assertEquals("[]", authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText()) + } + + @Test + fun `bulk membership addition changes nothing when a subscription is missing`() = withApp { + val subscribed = channel("subscribed") + subscriptions.add(TEST_USER_ID, SubscriptionItem(subscribed, "Subscribed", "")) + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NotFound, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$subscribed","${channel("missing")}"]}""") + }.status) + assertEquals("[]", authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText()) + } + @Test fun `group membership projection returns account scoped memberships with subscription data`() = withApp { val sharedChannel = channel("shared") From 27a3ff3e831b5309560e9ac413fd87a3ef6693af Mon Sep 17 00:00:00 2001 From: User Date: Tue, 25 Aug 2026 18:24:22 -0700 Subject: [PATCH 64/68] fix: reject ambiguous subscription group deletion Return a client error when a DELETE supplies both the legacy URL query parameter and a JSON membership body so the server cannot silently apply only part of the requested mutation. Constraint: Preserve both shipped query-only deletion and the new body-only batch contract Rejected: Give the query parameter precedence | silently ignores a valid batch body Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep the two DELETE input forms mutually exclusive at the HTTP boundary Tested: Red-green route regression, JDK 25 check with 1,135 tests, coverage, shadowJar, OpenAPI validation, and live HTTP QA Not-tested: Frontend integration remains intentionally deferred --- openapi/paths/subscriptions.yaml | 2 +- .../server/routes/SubscriptionGroupsRoutes.kt | 7 +++++++ .../server/SubscriptionGroupsRoutesTest.kt | 20 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index 7783c19c..a765c764 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -149,7 +149,7 @@ SubscriptionGroupChannels: delete: tags: [user-data] summary: Remove subscribed channels from a group - description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. + description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. Supplying both url and a request body returns 400. parameters: - name: url in: query diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt index be9f93c5..452a994d 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -10,6 +10,7 @@ import dev.typetype.server.services.SubscriptionGroupsService import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive +import io.ktor.server.request.receiveText import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.delete @@ -60,6 +61,12 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() val queryChannelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) if (queryChannelUrl != null) { + if (call.receiveText().isNotBlank()) { + return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Specify either url or a request body, not both"), + ) + } return@withJwtAuth call.respondMembership( groupsService.removeSubscription(userId, groupId, queryChannelUrl), ) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt index 46ebea82..8ba7517c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -162,6 +162,26 @@ class SubscriptionGroupsRoutesTest { assertEquals("[]", authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText()) } + @Test + fun `membership deletion rejects query and body together`() = withApp { + val first = channel("one") + val second = channel("two") + listOf(first, second).forEach { url -> + subscriptions.add(TEST_USER_ID, SubscriptionItem(url, url.substringAfterLast('/'), "")) + } + val group = createGroup("Work") + listOf(first, second).forEach { url -> groups.addSubscription(TEST_USER_ID, group.id, url) } + + assertEquals(HttpStatusCode.BadRequest, client.delete("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + parameter("url", first) + setBody("""{"channelUrls":["$second"]}""") + }.status) + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText() + assertTrue(grouped.contains(first)) + assertTrue(grouped.contains(second)) + } + @Test fun `bulk membership addition changes nothing when a subscription is missing`() = withApp { val subscribed = channel("subscribed") From ca23ed86d5f126bc49c7d260d8b9d3343f593c08 Mon Sep 17 00:00:00 2001 From: User Date: Tue, 25 Aug 2026 20:34:13 -0700 Subject: [PATCH 65/68] fix: bound subscription group membership requests Reject oversized membership bodies before they can be fully buffered and validate each submitted channel URL at the API boundary. Treat any non-empty DELETE body as present so whitespace cannot bypass query/body exclusivity. Constraint: Preserve the existing singular and batch membership contracts Rejected: Rely on Content-Length alone | chunked requests can omit the header Confidence: high Scope-risk: narrow Directive: Keep request limits aligned with the OpenAPI membership schemas Tested: ./gradlew --no-daemon clean check shadowJar validateOpenApi Tested: Live HTTP checks for body limits, URL length, and DELETE ambiguity --- openapi/components/subscriptions.yaml | 4 +- openapi/paths/subscriptions.yaml | 8 +- .../SubscriptionGroupMembershipRequestBody.kt | 85 +++++++++++ .../server/routes/SubscriptionGroupsRoutes.kt | 35 ++--- ...nGroupMembershipRequestLimitsRoutesTest.kt | 132 ++++++++++++++++++ 5 files changed, 233 insertions(+), 31 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index 20a7476c..d0af4c01 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -43,7 +43,7 @@ SubscriptionGroupMembershipRequest: type: object required: [channelUrl] properties: - channelUrl: { type: string, minLength: 1 } + channelUrl: { type: string, minLength: 1, maxLength: 2048 } SubscriptionGroupMembershipBatchRequest: type: object required: [channelUrls] @@ -52,7 +52,7 @@ SubscriptionGroupMembershipBatchRequest: type: array minItems: 1 maxItems: 500 - items: { type: string, minLength: 1 } + items: { type: string, minLength: 1, maxLength: 2048 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index a765c764..0a6244ab 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -132,7 +132,7 @@ SubscriptionGroupChannels: put: tags: [user-data] summary: Add subscribed channels to a group - description: Accepts the original singular channelUrl request or up to 500 channelUrls. Batch additions are atomic, canonicalized and deduplicated. + description: Accepts the original singular channelUrl request or up to 500 channelUrls. Batch additions are atomic, canonicalized and deduplicated. Request bodies are limited to 1 MiB. requestBody: required: true content: @@ -145,16 +145,17 @@ SubscriptionGroupChannels: '204': { description: Membership exists. } '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } delete: tags: [user-data] summary: Remove subscribed channels from a group - description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. Supplying both url and a request body returns 400. + description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. Supplying both url and any non-empty request body returns 400. Request bodies are limited to 1 MiB. parameters: - name: url in: query required: false - schema: { type: string, minLength: 1 } + schema: { type: string, minLength: 1, maxLength: 2048 } requestBody: required: false content: @@ -167,6 +168,7 @@ SubscriptionGroupChannels: '204': { description: Membership deleted. } '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } SubscriptionFeed: get: diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt new file mode 100644 index 00000000..022aeaa6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionGroupMembershipRequest +import dev.typetype.server.services.SubscriptionGroupsService +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.contentType +import io.ktor.server.request.receiveChannel +import io.ktor.server.response.respond +import io.ktor.utils.io.ByteReadChannel +import io.ktor.utils.io.jvm.javaio.toInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import java.io.ByteArrayOutputStream + +private const val MAX_MEMBERSHIP_REQUEST_BODY_BYTES = 1024 * 1024 +private const val MAX_MEMBERSHIP_CHANNEL_URL_LENGTH = 2048 +private val membershipRequestJson = Json { ignoreUnknownKeys = true } + +internal sealed interface MembershipChannels { + data class Single(val channelUrl: String) : MembershipChannels + data class Batch(val channelUrls: List) : MembershipChannels +} + +internal suspend fun ApplicationCall.receiveMembershipChannels(body: ByteArray): MembershipChannels? { + val request = if (request.contentType().match(ContentType.Application.Json)) { + try { + membershipRequestJson.decodeFromString(body.decodeToString()) + } catch (_: SerializationException) { + null + } + } else { + null + } + val channelUrl = request?.channelUrl?.takeIf(String::isValidMembershipChannelUrl) + val channelUrls = request?.channelUrls + val parsed = when { + channelUrl != null && channelUrls == null -> MembershipChannels.Single(channelUrl) + request?.channelUrl == null && channelUrls != null && + channelUrls.size in 1..SubscriptionGroupsService.MAX_MEMBERSHIP_CHANNELS && + channelUrls.all(String::isValidMembershipChannelUrl) -> MembershipChannels.Batch(channelUrls) + else -> null + } + if (parsed == null) respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + return parsed +} + +internal suspend fun ApplicationCall.receiveMembershipBody(): ByteArray? { + val contentLength = request.headers[HttpHeaders.ContentLength]?.toLongOrNull() + if (contentLength != null && contentLength > MAX_MEMBERSHIP_REQUEST_BODY_BYTES) { + respondMembershipBodyTooLarge() + return null + } + val body = receiveChannel().readUpTo(MAX_MEMBERSHIP_REQUEST_BODY_BYTES) + if (body == null) respondMembershipBodyTooLarge() + return body +} + +private suspend fun ByteReadChannel.readUpTo(maxBytes: Int): ByteArray? = withContext(Dispatchers.IO) { + toInputStream().use { input -> + ByteArrayOutputStream().use { output -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read <= 0) break + if (output.size() + read > maxBytes) return@withContext null + output.write(buffer, 0, read) + } + output.toByteArray() + } + } +} + +internal fun String.isValidMembershipChannelUrl(): Boolean = + isNotBlank() && length <= MAX_MEMBERSHIP_CHANNEL_URL_LENGTH + +private suspend fun ApplicationCall.respondMembershipBodyTooLarge() = respond( + HttpStatusCode.PayloadTooLarge, + ErrorResponse("Request body exceeds 1 MiB", "request_body_too_large"), +) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt index 452a994d..2109f4b7 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -1,7 +1,6 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.models.SubscriptionGroupMembershipRequest import dev.typetype.server.models.SubscriptionGroupRequest import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionGroupMembershipResult @@ -10,7 +9,6 @@ import dev.typetype.server.services.SubscriptionGroupsService import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive -import io.ktor.server.request.receiveText import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.delete @@ -46,7 +44,8 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut put("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - when (val request = call.receiveMembershipChannels() ?: return@withJwtAuth) { + val body = call.receiveMembershipBody() ?: return@withJwtAuth + when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { is MembershipChannels.Single -> call.respondMembership( groupsService.addSubscription(userId, groupId, request.channelUrl), ) @@ -59,9 +58,13 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut delete("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - val queryChannelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) + val queryChannelUrl = call.request.queryParameters["url"] + if (queryChannelUrl != null && !queryChannelUrl.isValidMembershipChannelUrl()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid channel URL")) + } + val body = call.receiveMembershipBody() ?: return@withJwtAuth if (queryChannelUrl != null) { - if (call.receiveText().isNotBlank()) { + if (body.isNotEmpty()) { return@withJwtAuth call.respond( HttpStatusCode.BadRequest, ErrorResponse("Specify either url or a request body, not both"), @@ -71,7 +74,7 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut groupsService.removeSubscription(userId, groupId, queryChannelUrl), ) } - when (val request = call.receiveMembershipChannels() ?: return@withJwtAuth) { + when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { is MembershipChannels.Single -> call.respondMembership( groupsService.removeSubscription(userId, groupId, request.channelUrl), ) @@ -91,26 +94,6 @@ private suspend fun ApplicationCall.receiveGroupRequest(): SubscriptionGroupRequ null } -private sealed interface MembershipChannels { - data class Single(val channelUrl: String) : MembershipChannels - data class Batch(val channelUrls: List) : MembershipChannels -} - -private suspend fun ApplicationCall.receiveMembershipChannels(): MembershipChannels? { - val request = runCatching { receive() }.getOrNull() - val channelUrl = request?.channelUrl?.takeIf(String::isNotBlank) - val channelUrls = request?.channelUrls - val parsed = when { - channelUrl != null && channelUrls == null -> MembershipChannels.Single(channelUrl) - request?.channelUrl == null && channelUrls != null && - channelUrls.size in 1..SubscriptionGroupsService.MAX_MEMBERSHIP_CHANNELS && - channelUrls.all(String::isNotBlank) -> MembershipChannels.Batch(channelUrls) - else -> null - } - if (parsed == null) respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) - return parsed -} - private suspend fun ApplicationCall.respondGroupWrite(result: SubscriptionGroupWriteResult, created: Boolean) { when (result) { is SubscriptionGroupWriteResult.Success -> if (created) respond(HttpStatusCode.Created, result.group) else { diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt new file mode 100644 index 00000000..1b599dec --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt @@ -0,0 +1,132 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionGroupsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.OutgoingContent +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.ktor.utils.io.ByteWriteChannel +import io.ktor.utils.io.writeByteArray +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupMembershipRequestLimitsRoutesTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + private const val EXPECTED_BODY_LIMIT_BYTES = 1024 * 1024 + private const val EXPECTED_CHANNEL_URL_LIMIT = 2048 + + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `membership mutation rejects declared body over one mebibyte`() = withApp { + val group = createGroup() + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("x".repeat(EXPECTED_BODY_LIMIT_BYTES + 1)) + } + + assertEquals(HttpStatusCode.PayloadTooLarge, response.status) + } + + @Test + fun `membership mutation rejects streamed body over one mebibyte`() = withApp { + val group = createGroup() + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorize() + setBody(oversizedStreamingBody()) + } + + assertEquals(HttpStatusCode.PayloadTooLarge, response.status) + } + + @Test + fun `membership mutation rejects channel urls over 2048 characters`() = withApp { + val group = createGroup() + val channelUrl = "https://example.com/" + "a".repeat(EXPECTED_CHANNEL_URL_LIMIT) + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"$channelUrl"}""") + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `membership deletion rejects query and whitespace body together`() = withApp { + val channelUrl = "https://example.com/channel" + subscriptions.add(TEST_USER_ID, SubscriptionItem(channelUrl, "Channel", "")) + val group = createGroup() + groups.addSubscription(TEST_USER_ID, group.id, channelUrl) + + val response = client.delete("/subscriptions/groups/${group.id}/channels") { + authorize() + parameter("url", channelUrl) + header(HttpHeaders.ContentType, ContentType.Text.Plain.toString()) + setBody(" ") + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertEquals(1, groups.getAll(TEST_USER_ID).single().channelCount) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionGroupsRoutes(groups, auth) } + } + block() + } + + private suspend fun createGroup() = requireNotNull( + (groups.create(TEST_USER_ID, "Work") as? SubscriptionGroupWriteResult.Success)?.group, + ) + + private fun oversizedStreamingBody() = object : OutgoingContent.WriteChannelContent() { + override val contentType = ContentType.Application.Json + + override suspend fun writeTo(channel: ByteWriteChannel) { + val chunk = ByteArray(64 * 1024) { 'x'.code.toByte() } + repeat(EXPECTED_BODY_LIMIT_BYTES / chunk.size + 1) { channel.writeByteArray(chunk) } + } + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorizeJson() { + authorize() + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } +} From bdfe561a7d8fb298a73b16ad6d12d7258458d7a8 Mon Sep 17 00:00:00 2001 From: User Date: Wed, 26 Aug 2026 12:41:44 -0700 Subject: [PATCH 66/68] fix: keep subscription membership reads consistent Repair avatars only after applying subscription selection so unrelated channels neither consume the repair budget nor receive database writes. Serialize membership projections with account subscription mutations so channel data and group assignments come from one coherent state. Constraint: Preserve avatar repair for unfiltered and membership projection responses Rejected: Repair all subscriptions before filtering | unrelated rows consume the repair limit and receive writes Rejected: Add a new snapshot transaction API | the existing per-user mutation lock already serializes group changes Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep membership projections and mutations on SubscriptionMutationLock Tested: JDK 25 clean check, 1,141 tests, shadowJar, OpenAPI validation, and live HTTP lock-contention QA --- .../server/services/SubscriptionsService.kt | 9 ++--- .../server/SubscriptionGroupsServiceTest.kt | 34 ++++++++++++++++++ .../SubscriptionsAvatarRepairServiceTest.kt | 36 +++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 70da17df..b0bb5dd9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -21,12 +21,14 @@ class SubscriptionsService { selection: SubscriptionSelection = SubscriptionSelection.All, ): List = DatabaseFactory.query { val selectedUrls = selectedChannelUrls(userId, selection) - subscriptionItems(userId) + val selectedItems = subscriptionItems(userId) .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } + SubscriptionAvatarRepairer.repair(userId = userId, items = selectedItems) } suspend fun getAllWithGroupMemberships(userId: String): List = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val groupIdsByChannel = SubscriptionGroupMembershipsTable .innerJoin(SubscriptionGroupsTable) .selectAll() @@ -41,7 +43,7 @@ class SubscriptionsService { valueTransform = { it[SubscriptionGroupMembershipsTable.groupId] }, ) .mapValues { (_, groupIds) -> groupIds.sorted() } - subscriptionItems(userId).map { item -> + SubscriptionAvatarRepairer.repair(userId = userId, items = subscriptionItems(userId)).map { item -> SubscriptionGroupMembershipItem( channelUrl = item.channelUrl, name = item.name, @@ -81,11 +83,10 @@ class SubscriptionsService { } private fun subscriptionItems(userId: String): List { - val items = SubscriptionsTable.selectAll() + return SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) .map { it.toItem() } - return SubscriptionAvatarRepairer.repair(userId = userId, items = items) } private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index 4233deae..7f017a97 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -212,6 +212,40 @@ class SubscriptionGroupsServiceTest { assertTrue(allWaited, "all group mutations must wait for the account-scoped lock") } + @Test + fun `membership projection shares the account subscription lock`() = runTest { + val userId = "projection-user" + val group = groups.create(userId, "Group").createdGroup() + subscriptions.add(userId, subscription("one")) + groups.addSubscription(userId, group.id, channel("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val projection = async(Dispatchers.IO) { subscriptions.getAllWithGroupMemberships(userId) } + val readWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + while (!projection.isCompleted && waitingSubscriptionLocks(userId) == 0) yield() + !projection.isCompleted + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assertTrue(readWaited, "the membership projection must wait for the account-scoped lock") + assertEquals(listOf(group.id), projection.await().single().groupIds) + } + @Test fun `replacement imports retain only memberships for subscriptions still present`() = runTest { val group = groups.create("user", "Group").createdGroup() diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index d0f988c8..de4341ef 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -3,12 +3,19 @@ package dev.typetype.server import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.FavoritesTable import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.SubscriptionAvatarRepairer +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SubscriptionsService import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -16,6 +23,7 @@ import org.junit.jupiter.api.Test class SubscriptionsAvatarRepairServiceTest { private val service = SubscriptionsService() + private val groups = SubscriptionGroupsService() companion object { const val WATCH_CHANNEL_URL = "https://www.youtube.com/channel/UCWatch" @@ -62,6 +70,28 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(26, second.count { it.avatarUrl.isNotBlank() }) } + @Test + fun `filtered getAll repairs avatars only for selected subscriptions`() = runTest { + repeat(25) { index -> + val channelUrl = "https://www.youtube.com/channel/UCUnselected$index" + addSubscription(channelUrl) + addHistory( + channelUrl = channelUrl, + avatarUrl = "https://avatar.test/unselected-$index.jpg", + watchedAt = (index + 1).toLong(), + ) + } + addSubscription(WATCH_CHANNEL_URL) + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL, watchedAt = 0) + val group = (groups.create(TEST_USER_ID, "Selected") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, WATCH_CHANNEL_URL) + + val selected = service.getAll(TEST_USER_ID, SubscriptionSelection.Group(group.id)).single() + + assertEquals(WATCH_AVATAR_URL, selected.avatarUrl) + assertEquals("", storedAvatar("https://www.youtube.com/channel/UCUnselected0")) + } + @Test fun `avatar repair scans past unrepairable empty subscriptions`() = runTest { addWatchLater(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL) @@ -78,6 +108,12 @@ class SubscriptionsAvatarRepairServiceTest { service.add(TEST_USER_ID, SubscriptionItem(channelUrl = channelUrl, name = "Channel", avatarUrl = "")) } + private suspend fun storedAvatar(channelUrl: String): String = DatabaseFactory.query { + SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq TEST_USER_ID) and (SubscriptionsTable.channelUrl eq channelUrl) + }.single()[SubscriptionsTable.avatarUrl] + } + private suspend fun addWatchLater(channelUrl: String, avatarUrl: String): Unit = DatabaseFactory.query { WatchLaterTable.insert { it[userId] = TEST_USER_ID; it[url] = "https://video.test/watch"; it[title] = "Video"; it[thumbnail] = "" From b24b165ba79be405e4caa9fd939409e1e3a51e98 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 27 Aug 2026 07:14:53 +0200 Subject: [PATCH 67/68] fix: preserve SABR rewind preparation --- .../kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt | 2 +- .../dev/typetype/server/services/SabrTargetRequestShape.kt | 3 ++- .../dev/typetype/server/services/SabrSeekRepositionPumpTest.kt | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 1f3953ed..3a5fecd7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -183,7 +183,7 @@ internal class SabrSessionPumpLoop( if (holder.isHistoricalLiveRequest(request)) { holder.setPlaybackState(SabrPlaybackState.REPOSITIONING) holder.prepareForHistoricalLiveRewind(request) - return withTargetedRequestShape(holder, request) { + return withTargetedRequestShape(holder, request, prepareSession = false) { pumpUntilCached(holder, localization, request, runtime) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt index 0b14da86..6c7bbb77 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory internal inline fun withTargetedRequestShape( holder: SabrSessionHolder, request: SabrSegmentRequest, + prepareSession: Boolean = true, block: () -> T, ): T { val companion = holder.companionFormat(request.format) @@ -16,7 +17,7 @@ internal inline fun withTargetedRequestShape( val requestStartMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) val targetPlayerTimeMs = request.targetPlayerTimeMs(holder, requestStartMs) val ranges = listOf(request.targetRange(holder), companion.targetCompanionRange(holder, targetPlayerTimeMs)) - holder.session.prepareForMediaSegment(request) + if (prepareSession) holder.session.prepareForMediaSegment(request) state.setPlayerTimeMs(targetPlayerTimeMs) state.setRequestTrackMode(request.trackMode(), true, true) state.setSelectVideoFormatBeforeAudio(request.format.isAudio) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index f07ad191..25c24a4c 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -170,7 +170,7 @@ class SabrSeekRepositionPumpTest { SabrSessionPumpLoop().run({ rounds++ < 1 }, holder, intervalMs = 0L) verify(exactly = 1) { session.prepareForRewind(request) } - verify(exactly = 1) { session.prepareForMediaSegment(request) } + verify(exactly = 0) { session.prepareForMediaSegment(request) } verify(exactly = 1) { state.setBufferedRangesOverride(null) } verify(exactly = 1) { session.pumpOnceStreamingForDemand(any(), request) } } finally { From 744d26dad132aa3c086f8ad3c14262efee9ffb2b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 28 Aug 2026 16:41:13 +0200 Subject: [PATCH 68/68] chore: prepare server 1.7.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index aa33b2fc..64006a63 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.6.0 +appVersion=1.7.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000