diff --git a/build.gradle.kts b/build.gradle.kts index 406be47e..620ef3ae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,8 +25,9 @@ repositories { } dependencies { - implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) - implementation(platform("io.netty:netty-bom:4.2.16.Final")) + implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) + implementation("com.fasterxml.jackson.core:jackson-core") + implementation(platform("io.netty:netty-bom:4.2.17.Final")) constraints { implementation("org.jsoup:jsoup:1.23.1") { because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") @@ -42,12 +43,12 @@ dependencies { implementation("io.ktor:ktor-server-status-pages-jvm") implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") - implementation("ch.qos.logback:logback-classic:1.6.1") + implementation("ch.qos.logback:logback-classic:1.6.3") implementation("com.github.Priveetee.PipePipeExtractor:extractor:f156813dd4bbebf3b4dffe541fee6c27ae1dd294") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("org.json:json:20260719") - implementation("com.squareup.okhttp3:okhttp:5.4.0") - implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") + implementation("org.json:json:20260814") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.4.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") implementation("com.zaxxer:HikariCP:7.1.0") diff --git a/gradle.properties b/gradle.properties index aa33b2fc..64006a63 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.6.0 +appVersion=1.7.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index 37be9bbe..63aec2da 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -17,6 +17,7 @@ tags: - name: youtube-session - name: user-data - name: rss + - name: portability paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } @@ -44,6 +45,7 @@ paths: /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } + /subscriptions/group-memberships: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupMemberships } /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } @@ -56,6 +58,14 @@ paths: /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } + /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats } + /portability/imports: { $ref: ./openapi/paths/portability.yaml#/PortabilityImports } + /portability/exports: { $ref: ./openapi/paths/portability.yaml#/PortabilityExports } + /portability/jobs/{id}: { $ref: ./openapi/paths/portability.yaml#/PortabilityJob } + /portability/jobs/{id}/apply: { $ref: ./openapi/paths/portability.yaml#/PortabilityApply } + /portability/jobs/{id}/cancel: { $ref: ./openapi/paths/portability.yaml#/PortabilityCancel } + /portability/jobs/{id}/report: { $ref: ./openapi/paths/portability.yaml#/PortabilityReport } + /portability/jobs/{id}/artifact: { $ref: ./openapi/paths/portability.yaml#/PortabilityArtifact } /blocked/keywords: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeywords } /blocked/keywords/{keyword}: { $ref: ./openapi/paths/user-backup.yaml#/BlockedKeyword } /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } @@ -151,9 +161,11 @@ components: SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } SubscriptionItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionItem } + SubscriptionGroupMembershipItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipItem } SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + SubscriptionGroupMembershipBatchRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } @@ -163,6 +175,11 @@ components: SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } + PortabilityAdapterDescriptor: { $ref: ./openapi/components/portability.yaml#/PortabilityAdapterDescriptor } + PortabilityJobSnapshot: { $ref: ./openapi/components/portability.yaml#/PortabilityJobSnapshot } + PortabilityJobReport: { $ref: ./openapi/components/portability.yaml#/PortabilityJobReport } + PortabilityImportRequest: { $ref: ./openapi/components/portability.yaml#/PortabilityImportRequest } + PortabilityExportRequest: { $ref: ./openapi/components/portability.yaml#/PortabilityExportRequest } BlockedKeywordItem: { $ref: ./openapi/components/user-backup.yaml#/BlockedKeywordItem } AdminSettingsItem: { $ref: ./openapi/components/access-control.yaml#/AdminSettingsItem } AllowedChannelItem: { $ref: ./openapi/components/access-control.yaml#/AllowedChannelItem } diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index a0629e47..c40a33cc 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,6 +20,7 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady + - parentalControlsEnabled - rss properties: name: { type: string, example: TypeType } @@ -50,6 +51,9 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] + parentalControlsEnabled: + type: boolean + description: True when the instance-wide allow-list policy is enabled. rss: $ref: '#/RssInstanceCapability' RssInstanceCapability: diff --git a/openapi/components/portability.yaml b/openapi/components/portability.yaml new file mode 100644 index 00000000..05302336 --- /dev/null +++ b/openapi/components/portability.yaml @@ -0,0 +1,125 @@ +PortabilityFormat: + type: string + enum: [typetype, pipepipe, newpipe, invidious, piped, libretube, viewtube, materialious, youtube-local, flow, skytube, grayjay, youtube-takeout, opml] +PortabilityCategory: + type: string + enum: [subscriptions, subscriptionGroups, history, playlists, watchLater, favorites, progress, searchHistory, savedPlaylists, settings, contentFilters] +PortabilityCapability: + type: object + required: [category, directions, fidelity] + properties: + category: { $ref: '#/PortabilityCategory' } + directions: + type: array + uniqueItems: true + items: { type: string, enum: [import, export] } + fidelity: { type: string, enum: [complete, partial] } +PortabilityAdapterDescriptor: + type: object + required: [format, adapterVersion, capabilities, defaultExtension, contentType] + properties: + format: { $ref: '#/PortabilityFormat' } + adapterVersion: { type: integer, minimum: 1 } + capabilities: + type: array + uniqueItems: true + items: { $ref: '#/PortabilityCapability' } + defaultExtension: { type: string } + contentType: { type: string } +PortabilityIssue: + type: object + required: [category, code, message, count] + properties: + category: + allOf: [{ $ref: '#/PortabilityCategory' }] + nullable: true + code: { type: string } + message: { type: string } + count: { type: integer, format: int64, minimum: 1 } +PortabilityDetection: + type: object + required: [format, formatVersion, adapterVersion, confidence, evidence] + properties: + format: { $ref: '#/PortabilityFormat' } + formatVersion: { type: string, nullable: true } + adapterVersion: { type: integer, minimum: 1 } + confidence: { type: integer, minimum: 1, maximum: 100 } + evidence: { type: string } +PortabilityPreview: + type: object + required: [detection, counts, duplicates, issues] + properties: + detection: { $ref: '#/PortabilityDetection' } + counts: + type: object + additionalProperties: { type: integer, format: int64, minimum: 0 } + duplicates: { type: integer, format: int64, minimum: 0 } + issues: + type: array + items: { $ref: '#/PortabilityIssue' } +PortabilityJobSnapshot: + type: object + required: [id, kind, state, createdAt, updatedAt] + properties: + id: { type: string, format: uuid } + kind: { type: string, enum: [import, export] } + state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } + requestId: { type: string, nullable: true } + preview: + allOf: [{ $ref: '#/PortabilityPreview' }] + nullable: true + result: + type: object + nullable: true + additionalProperties: { type: integer, format: int64, minimum: 0 } + progress: + allOf: [{ $ref: '#/PortabilityJobProgress' }] + nullable: true + errorCode: { type: string, nullable: true } + errorMessage: { type: string, nullable: true } +PortabilityJobProgress: + type: object + required: [phase, unit, processed, total] + properties: + phase: { type: string, enum: [analyzing, collecting, applying, encoding] } + unit: { type: string, enum: [records, categories, bytes] } + processed: { type: integer, format: int64, minimum: 0 } + total: { type: integer, format: int64, minimum: 0, nullable: true } +PortabilityJobReport: + type: object + required: [id, state] + properties: + id: { type: string, format: uuid } + state: { type: string, enum: [queued, analyzing, ready, applying, encoding, completed, failed, cancelled] } + requestId: { type: string, nullable: true } + preview: + allOf: [{ $ref: '#/PortabilityPreview' }] + nullable: true + result: + type: object + nullable: true + additionalProperties: { type: integer, format: int64, minimum: 0 } + errorCode: { type: string, nullable: true } + errorMessage: { type: string, nullable: true } +PortabilityImportRequest: + type: object + required: [categories] + properties: + categories: + type: array + minItems: 1 + uniqueItems: true + items: { $ref: '#/PortabilityCategory' } + duplicatePolicy: { type: string, enum: [skip, replace], default: skip } +PortabilityExportRequest: + type: object + required: [format, categories] + properties: + format: { $ref: '#/PortabilityFormat' } + categories: + type: array + minItems: 1 + uniqueItems: true + items: { $ref: '#/PortabilityCategory' } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index 5894cc5f..d0af4c01 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -6,6 +6,18 @@ SubscriptionItem: name: { type: string } avatarUrl: { type: string } subscribedAt: { type: integer, format: int64 } +SubscriptionGroupMembershipItem: + type: object + required: [channelUrl, name, avatarUrl, subscribedAt, groupIds] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } + subscribedAt: { type: integer, format: int64 } + groupIds: + type: array + uniqueItems: true + items: { type: string, format: uuid } SubscriptionCreateRequest: type: object required: [channelUrl, name, avatarUrl] @@ -31,7 +43,16 @@ SubscriptionGroupMembershipRequest: type: object required: [channelUrl] properties: - channelUrl: { type: string, minLength: 1 } + channelUrl: { type: string, minLength: 1, maxLength: 2048 } +SubscriptionGroupMembershipBatchRequest: + type: object + required: [channelUrls] + properties: + channelUrls: + type: array + minItems: 1 + maxItems: 500 + items: { type: string, minLength: 1, maxLength: 2048 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/paths/downloader.yaml b/openapi/paths/downloader.yaml index 5b061cef..3838e4dc 100644 --- a/openapi/paths/downloader.yaml +++ b/openapi/paths/downloader.yaml @@ -89,6 +89,16 @@ JobArtifact: '200': { description: Artifact bytes. } '206': { description: Partial artifact bytes. } '302': { description: Redirect to public artifact storage. } + '416': { description: Requested byte range is outside the artifact. } + '404': { $ref: ../components/common.yaml#/JsonError } + head: + tags: [downloader] + summary: Inspect a completed artifact without transferring its body + responses: + '200': { description: Artifact metadata. } + '206': { description: Partial artifact metadata. } + '302': { description: Redirect to public artifact storage. } + '416': { description: Requested byte range is outside the artifact. } '404': { $ref: ../components/common.yaml#/JsonError } JobCancel: parameters: diff --git a/openapi/paths/portability.yaml b/openapi/paths/portability.yaml new file mode 100644 index 00000000..060de475 --- /dev/null +++ b/openapi/paths/portability.yaml @@ -0,0 +1,148 @@ +PortabilityFormats: + get: + tags: [portability] + summary: List supported portability adapters + responses: + '200': + description: Adapter capabilities and fidelity + content: + application/json: + schema: + type: array + items: { $ref: ../components/portability.yaml#/PortabilityAdapterDescriptor } + '401': { description: Missing or invalid token } + '403': { description: Guest account } +PortabilityImports: + post: + tags: [portability] + summary: Upload and analyze an account backup + parameters: + - name: format + in: query + required: false + description: Optional explicit source format. The engine auto-detects when omitted. + schema: { $ref: ../components/portability.yaml#/PortabilityFormat } + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: { type: string, format: binary } + responses: + '202': + description: Analysis job created + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Invalid or unrecognized backup } + '401': { description: Missing or invalid token } + '413': { description: Upload exceeds the configured limit } +PortabilityExports: + post: + tags: [portability] + summary: Create an account export + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityExportRequest } + responses: + '202': + description: Export job created + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Unsupported format or category } + '401': { description: Missing or invalid token } +PortabilityJob: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Read an owned portability job + responses: + '200': + description: Current job state + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } + delete: + tags: [portability] + summary: Delete an owned portability job and its temporary files + responses: + '204': { description: Job deleted } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } +PortabilityApply: + parameters: + - $ref: '#/JobId' + post: + tags: [portability] + summary: Apply selected previewed categories + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityImportRequest } + responses: + '202': + description: Transactional import started + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '400': { description: Invalid category selection } + '409': { description: Job is not ready } +PortabilityCancel: + parameters: + - $ref: '#/JobId' + post: + tags: [portability] + summary: Cancel an active portability job + responses: + '200': + description: Cancelled or already terminal job + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot } + '404': { description: Job not found for this account } +PortabilityReport: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Read the durable result and compatibility report for an owned job + responses: + '200': + description: Current or final portability report + content: + application/json: + schema: { $ref: ../components/portability.yaml#/PortabilityJobReport } + '401': { description: Missing or invalid token } + '404': { description: Job not found for this account } +PortabilityArtifact: + parameters: + - $ref: '#/JobId' + get: + tags: [portability] + summary: Download a completed export artifact + responses: + '200': + description: Generated export + headers: + Content-Disposition: + schema: { type: string } + content: + application/octet-stream: + schema: { type: string, format: binary } + '404': { description: Job not found for this account } + '409': { description: Export is not complete } +JobId: + name: id + in: path + required: true + schema: { type: string, format: uuid } diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index 9f9b4863..0a6244ab 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -52,6 +52,20 @@ Subscriptions: '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroupMemberships: + get: + tags: [user-data] + summary: List subscriptions with their group memberships + description: Returns every current subscription once, with all account-owned group IDs assigned to that channel. Ungrouped subscriptions have an empty groupIds array. + responses: + '200': + description: Account-scoped subscriptions and their complete group memberships. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipItem } + '401': { $ref: ../components/common.yaml#/JsonError } SubscriptionGroups: get: tags: [user-data] @@ -117,29 +131,44 @@ SubscriptionGroupChannels: schema: { type: string, format: uuid } put: tags: [user-data] - summary: Add a subscribed channel to a group + summary: Add subscribed channels to a group + description: Accepts the original singular channelUrl request or up to 500 channelUrls. Batch additions are atomic, canonicalized and deduplicated. Request bodies are limited to 1 MiB. requestBody: required: true content: application/json: - schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } responses: '204': { description: Membership exists. } '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } delete: tags: [user-data] - summary: Remove a subscribed channel from a group + summary: Remove subscribed channels from a group + description: The url query parameter preserves the original singular contract. A JSON request body can remove one channel or up to 500 channelUrls atomically; absent batch memberships are ignored. Supplying both url and any non-empty request body returns 400. Request bodies are limited to 1 MiB. parameters: - name: url in: query - required: true - schema: { type: string, minLength: 1 } + required: false + schema: { type: string, minLength: 1, maxLength: 2048 } + requestBody: + required: false + content: + application/json: + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } responses: '204': { description: Membership deleted. } '400': { $ref: ../components/common.yaml#/JsonError } '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } '404': { $ref: ../components/common.yaml#/JsonError } SubscriptionFeed: get: diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 299d304c..05786bc2 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -24,8 +24,14 @@ import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserConfig import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.services.YoutubeRemoteLoginReadinessService +import dev.typetype.server.portability.PortabilityEngineFactory import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationStopped import io.ktor.server.netty.EngineMain +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import java.nio.file.Files import java.util.UUID fun main(args: Array) = EngineMain.main(args) @@ -83,6 +89,11 @@ fun Application.module() { val downloaderGatewayService = DownloaderGatewayService(downloaderServiceUrl) val openMojiProxyService = OpenMojiProxyService(cache) val internalHealthService = InternalHealthService(cache, downloaderGatewayService, subtitleServiceUrl) + val portabilityEngine = PortabilityEngineFactory.create( + Files.createTempDirectory("typetype-portability-"), + CoroutineScope(SupervisorJob() + Dispatchers.IO), + ) + monitor.subscribe(ApplicationStopped) { portabilityEngine.close() } configurePlugins(authService) installApplicationRoutes( svc = svc, @@ -102,5 +113,6 @@ fun Application.module() { internalHealthService = internalHealthService, restoreService = restoreService, youtubeRemoteBrowserService = youtubeRemoteBrowserService, + portabilityEngine = portabilityEngine, ) } diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 1976d9cf..3a930d92 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -42,6 +42,7 @@ import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService +import dev.typetype.server.portability.PortabilityEngine import io.ktor.server.application.Application import io.ktor.server.plugins.ratelimit.rateLimit import io.ktor.server.routing.routing @@ -64,6 +65,7 @@ internal fun Application.installApplicationRoutes( internalHealthService: InternalHealthService, restoreService: PipePipeBackupImporterService, youtubeRemoteBrowserService: YoutubeRemoteBrowserService, + portabilityEngine: PortabilityEngine, ) { routing { internalObservabilityRoutes(internalHealthService::check) @@ -117,6 +119,16 @@ internal fun Application.installApplicationRoutes( adminBugReportRoutes(authService, svc.bugReportService, gitHubIssueService) avatarRoutes(avatarService, openMojiProxyService, svc.customAvatarService) rateLimit(USER_DATA_ZONE) { youtubeRemoteBrowserRoutes(youtubeRemoteBrowserService, authService) } - rateLimit(USER_DATA_ZONE) { userDataRoutes(svc, authService, profileService, avatarService, svc.bugReportService, restoreService) } + rateLimit(USER_DATA_ZONE) { + userDataRoutes( + svc, + authService, + profileService, + avatarService, + svc.bugReportService, + restoreService, + portabilityEngine, + ) + } } } diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 5791d986..2dc6fd25 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -134,10 +134,8 @@ internal class ExtractionServiceRegistry( ), YouTubeSubtitleCache(cache), ) - val youtubeSabrBootstrapStreamService = CachedStreamService( - YoutubeScopedStreamService(SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient)), - cache, - "stream-youtube-sabr-bootstrap:v1", + val youtubeSabrBootstrapStreamService = YoutubeScopedStreamService( + SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient), ) val nicoNicoStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-niconico:v1") val bilibiliStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-bilibili:v1") diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt b/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt index c5e905b9..5f5e45ab 100644 --- a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt +++ b/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt @@ -27,6 +27,13 @@ data class BugReportContextItem( val timestamp: Long, val userAgent: String, val browserLanguage: String, + val viewportWidth: Int? = null, + val viewportHeight: Int? = null, + val screenWidth: Int? = null, + val screenHeight: Int? = null, + val devicePixelRatio: Double? = null, + val online: Boolean? = null, + val timezone: String? = null, val playerState: JsonElement? = null, val crashLogs: List = emptyList(), val apiErrors: List = emptyList(), diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 55d8916d..133514e6 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,6 +24,7 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, + val parentalControlsEnabled: Boolean = false, val rss: RssInstanceCapability = RssInstanceCapability(), ) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt new file mode 100644 index 00000000..b4a6fa60 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupMembershipItem( + val channelUrl: String, + val name: String, + val avatarUrl: String, + val subscribedAt: Long, + val groupIds: List, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt index 9a2fcb5f..d6befc62 100644 --- a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt @@ -3,4 +3,7 @@ package dev.typetype.server.models import kotlinx.serialization.Serializable @Serializable -data class SubscriptionGroupMembershipRequest(val channelUrl: String) +data class SubscriptionGroupMembershipRequest( + val channelUrl: String? = null, + val channelUrls: List? = null, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt new file mode 100644 index 00000000..1fa4b033 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class FlowPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.FLOW, + 2, + setOf( + flowCapability(PortabilityCategory.SUBSCRIPTIONS), + flowCapability(PortabilityCategory.SUBSCRIPTION_GROUPS), + flowCapability(PortabilityCategory.HISTORY), + flowCapability(PortabilityCategory.PROGRESS), + flowCapability(PortabilityCategory.SEARCH_HISTORY), + flowCapability(PortabilityCategory.FAVORITES), + flowCapability(PortabilityCategory.CONTENT_FILTERS, PortabilityFidelity.PARTIAL), + PortabilityCapability(PortabilityCategory.PLAYLISTS, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.PARTIAL), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (!probe.contains("\"viewHistory\"") || !probe.contains("\"subscriptionGroups\"")) return null + if (!probe.contains("\"playlistVideos\"") && !probe.contains("\"likedVideos\"")) return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.FLOW, version, 98, "Flow backup fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "version" -> version = parser.intValue + "subscriptions" -> FlowPortabilityReader.subscriptions(parser, sink, token) + "subscriptionGroups" -> FlowPortabilityReader.groups(parser, sink, token) + "viewHistory" -> FlowPortabilityReader.history(parser, sink, token) + "searchHistory" -> FlowPortabilityReader.searchHistory(parser, sink, token) + "playlists" -> FlowPortabilityReader.playlists(parser, sink, token) + "playlistVideos" -> FlowPortabilityReader.playlistVideos(parser, sink, token) + "likedVideos" -> FlowPortabilityReader.favorites(parser, sink, token) + "contentPreferences" -> FlowPortabilityReader.contentFilters(parser, sink, token) + else -> parser.skipChildren() + } + } + require(version == 2) { "Unsupported Flow backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = FlowPortabilityWriter.write(source, output, categories) +} + +private fun flowCapability( + category: PortabilityCategory, + fidelity: PortabilityFidelity = PortabilityFidelity.COMPLETE, +) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt new file mode 100644 index 00000000..ecc999e6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt @@ -0,0 +1,119 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.jsonObject + +internal object FlowPortabilityReader { + fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SUBSCRIPTIONS, parser, token) { item -> + val id = item.string("channelId") + if (id.isNotBlank()) { + sink.write( + PortabilitySubscription( + youtubeChannelUrl(id), + item.string("channelName"), + item.string("channelThumbnail"), + item.long("subscribedAt"), + ), + ) + } + } + + fun groups(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SUBSCRIPTION_GROUPS, parser, token) { item -> + val name = item.string("name") + if (name.isBlank()) return@readFlowArray + sink.write(PortabilitySubscriptionGroup(name)) + var memberships = 0 + item.string("channelIds").splitToSequence(',').map(String::trim).filter(String::isNotBlank).forEach { channel -> + require(memberships++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Flow group contains too many channels" } + sink.write(PortabilitySubscriptionGroupMembership(name, youtubeChannelUrl(channel))) + } + } + + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.markCategory(PortabilityCategory.PROGRESS) + sink.readFlowArray(PortabilityCategory.HISTORY, parser, token) { item -> + val id = item.string("videoId") + if (id.isBlank()) return@readFlowArray + val video = PortabilityVideo( + youtubeVideoUrl(id), + item.string("title"), + item.string("thumbnailUrl"), + item.long("duration") / 1_000L, + item.string("channelName"), + youtubeChannelUrl(item.string("channelId")), + ) + sink.write(PortabilityHistory(video, item.long("timestamp"), item.long("position") / 1_000L)) + sink.write(PortabilityProgress(video.url, item.long("position") / 1_000L, item.long("timestamp"))) + } + } + + fun searchHistory(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.SEARCH_HISTORY, parser, token) { item -> + val query = item.string("query") + if (query.isNotBlank()) sink.write(PortabilitySearchHistory(query, item.long("timestamp"))) + } + + fun playlists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.PLAYLISTS, parser, token) { item -> + val id = item.string("id") + if (id.isNotBlank()) sink.write(PortabilityPlaylist(id, item.string("name"), item.string("description"), item.long("createdAt"))) + } + + fun playlistVideos(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.PLAYLISTS, parser, token) { item -> + val playlistId = item.string("playlistId") + val videoId = item.string("videoId") + if (playlistId.isNotBlank() && videoId.isNotBlank()) { + sink.write( + PortabilityPlaylistVideo( + playlistId, + item.long("position").coerceIn(0L, Int.MAX_VALUE.toLong()).toInt(), + PortabilityVideo(youtubeVideoUrl(videoId)), + item.long("addedAt"), + ), + ) + } + } + + fun favorites(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = + sink.readFlowArray(PortabilityCategory.FAVORITES, parser, token) { item -> + val id = item.string("videoId") + if (id.isNotBlank()) { + sink.write( + PortabilityFavorite( + PortabilityVideo(youtubeVideoUrl(id), item.string("title"), item.string("thumbnail"), channelName = item.string("channelName")), + item.long("likedAt"), + ), + ) + } + } + + fun contentFilters(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_OBJECT) { "Invalid Flow content preferences" } + sink.markCategory(PortabilityCategory.CONTENT_FILTERS) + val item = parser.readJsonElement().jsonObject + item.array("blockedChannels").forEach { channel -> + val value = channel.stringValue() + if (value.isNotBlank()) sink.write(PortabilityContentFilter("blockedChannel", youtubeChannelUrl(value))) + } + } +} + +private fun PortabilityRecordSink.readFlowArray( + category: PortabilityCategory, + parser: JsonParser, + token: JsonToken, + block: (kotlinx.serialization.json.JsonObject) -> Unit, +) { + if (token == JsonToken.VALUE_NULL) return + require(token == JsonToken.START_ARRAY) { "Invalid Flow backup section" } + markCategory(category) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement().jsonObject) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt new file mode 100644 index 00000000..f087b3bf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt @@ -0,0 +1,132 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object FlowPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeNumberField("version", 2) + json.writeNumberField("timestamp", System.currentTimeMillis()) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + searchHistory(json, source, categories) + favorites(json, source, categories) + filters(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + json.writeStartObject() + json.writeStringField("channelId", youtubeId(item.channelUrl)) + json.writeStringField("channelName", item.name) + json.writeStringField("channelThumbnail", item.avatarUrl) + json.writeNumberField("subscribedAt", item.subscribedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptionGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val channels = boundedChannelIds(source, record.name) + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeStringField("channelIds", channels) + json.writeNumberField("sortOrder", 0) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun boundedChannelIds(source: PortabilityRecordSource, groupName: String): String { + val channels = StringBuilder() + var count = 0 + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, groupName) { child -> + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Flow group contains too many channels" } + val channel = youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl) + require(channels.length + channel.length + 1 <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { + "Flow group channel list is too large" + } + if (channels.isNotEmpty()) channels.append(',') + channels.append(channel) + } + return channels.toString() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("viewHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeNumberField("position", item.positionSeconds * 1_000L) + json.writeNumberField("duration", item.video.durationSeconds * 1_000L) + json.writeNumberField("timestamp", item.watchedAt) + json.writeStringField("title", item.video.title) + json.writeStringField("thumbnailUrl", item.video.thumbnailUrl) + json.writeStringField("channelName", item.video.channelName) + json.writeStringField("channelId", youtubeId(item.video.channelUrl)) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + val item = record as PortabilitySearchHistory + json.writeStartObject() + json.writeStringField("query", item.term) + json.writeNumberField("timestamp", item.searchedAt) + json.writeStringField("type", "TEXT") + json.writeEndObject() + } + json.writeEndArray() + } + + private fun favorites(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.FAVORITES !in categories) return + json.writeArrayFieldStart("likedVideos") + source.forEach(PortabilityCategory.FAVORITES) { record -> + val item = record as PortabilityFavorite + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("thumbnail", item.video.thumbnailUrl) + json.writeStringField("channelName", item.video.channelName) + json.writeNumberField("likedAt", item.favoritedAt) + json.writeBooleanField("isMusic", false) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun filters(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.CONTENT_FILTERS !in categories) return + json.writeObjectFieldStart("contentPreferences") + json.writeArrayFieldStart("blockedChannels") + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + val item = record as PortabilityContentFilter + if (item.kind == "blockedChannel") json.writeString(youtubeId(item.value)) + } + json.writeEndArray() + json.writeArrayFieldStart("preferredTopics") + json.writeEndArray() + json.writeArrayFieldStart("blockedTopics") + json.writeEndArray() + json.writeEndObject() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt new file mode 100644 index 00000000..14234e24 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt @@ -0,0 +1,65 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class GrayjayPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.GRAYJAY, + 1, + setOf( + grayjayCapability(PortabilityCategory.SUBSCRIPTIONS, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, BOTH, PortabilityFidelity.COMPLETE), + grayjayCapability(PortabilityCategory.HISTORY, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.PROGRESS, IMPORT, PortabilityFidelity.COMPLETE), + grayjayCapability(PortabilityCategory.PLAYLISTS, BOTH, PortabilityFidelity.PARTIAL), + grayjayCapability(PortabilityCategory.WATCH_LATER, BOTH, PortabilityFidelity.PARTIAL), + ), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + if ("exportInfo" !in archive.names || archive.names.none { it.startsWith("stores/") }) return null + val version = ZipFile(input.path.toFile()).use { zip -> + val entry = zip.getEntry("exportInfo") ?: return null + zip.getInputStream(entry).buffered().use { stream -> + Regex("\"version\"\\s*:\\s*\"([^\"]+)\"") + .find(stream.readNBytes(PortabilityLimits.PROBE_BYTES).decodeToString()) + ?.groupValues?.get(1) + } + } + if (version != "1") return null + return PortabilityDetection(PortabilityFormat.GRAYJAY, version, 99, "Grayjay exportInfo and stores") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported or encrypted Grayjay backup" } + ZipFile(input.path.toFile()).use { zip -> GrayjayPortabilityReader.read(zip, sink) } + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = super.assessExport(source, categories) + categories.mapNotNull { category -> + val fidelity = descriptor.capabilities.firstOrNull { it.category == category }?.fidelity + if (fidelity != PortabilityFidelity.PARTIAL) return@mapNotNull null + PortabilityIssue(category, "grayjay_partial_metadata", "Grayjay reconstructs this category from URLs, so some TypeType metadata is not represented") + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = GrayjayPortabilityWriter.write(source, output, categories) +} + +private val IMPORT = setOf(PortabilityDirection.IMPORT) +private val BOTH = setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT) + +private fun grayjayCapability( + category: PortabilityCategory, + directions: Set, + fidelity: PortabilityFidelity, +) = PortabilityCapability(category, directions, fidelity) diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt new file mode 100644 index 00000000..29fba1d8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt @@ -0,0 +1,101 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import java.util.zip.ZipFile + +internal object GrayjayPortabilityReader { + fun read(zip: ZipFile, sink: PortabilityRecordSink) { + readStrings(zip, "stores/subscriptions") { readSubscription(it, sink) } + readStrings(zip, "stores/subscription_groups") { readGroup(it, sink) } + readStrings(zip, "stores/history") { readHistory(it, sink) } + readStrings(zip, "stores/playlists") { readPlaylist(it, sink) } + readStrings(zip, "stores/watch_later") { readWatchLater(it, sink) } + } + + private fun readStrings(zip: ZipFile, name: String, consume: (String) -> Unit) { + val entry = zip.getEntry(name) ?: return + zip.getInputStream(entry).buffered().use { input -> + PortabilityJsonFactory.createParser(input).use { parser -> + require(parser.nextToken() == JsonToken.START_ARRAY) { "$name must contain an array" } + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "$name contains too many records" } + require(parser.currentToken() == JsonToken.VALUE_STRING) { "$name contains an invalid record" } + consume(parser.text) + } + } + } + } + + private fun readSubscription(value: String, sink: PortabilityRecordSink) { + val url = value.trim() + if (url.isNotBlank()) sink.write(PortabilitySubscription(url)) + } + + private fun readWatchLater(value: String, sink: PortabilityRecordSink) { + val url = value.trim() + if (url.isNotBlank()) sink.write(PortabilityWatchLater(PortabilityVideo(url))) + } + + private fun readHistory(value: String, sink: PortabilityRecordSink) { + val parts = value.split("|||", limit = 4) + if (parts.size != 4) { + sink.issue(PortabilityIssue(PortabilityCategory.HISTORY, "grayjay_history_invalid", "A Grayjay history record could not be parsed")) + return + } + val url = parts[0].trim() + val watchedAt = parts[1].toLongOrNull() ?: 0L + val position = parts[2].toLongOrNull() ?: 0L + if (url.isBlank()) return + sink.write(PortabilityHistory(PortabilityVideo(url, title = parts[3]), watchedAt, position)) + sink.write(PortabilityProgress(url, position, watchedAt)) + } + + private fun readPlaylist(value: String, sink: PortabilityRecordSink) { + val lines = value.lineSequence().filter(String::isNotBlank).toList() + if (lines.isEmpty()) return + val header = lines.first() + val separator = header.indexOf(":::") + val name = if (separator >= 0) header.substring(0, separator) else header + val sourceId = if (separator >= 0) header.substring(separator + 3).ifBlank { name } else name + sink.write(PortabilityPlaylist(sourceId, name)) + lines.drop(1).forEachIndexed { index, url -> + sink.write(PortabilityPlaylistVideo(sourceId, index, PortabilityVideo(url.trim()))) + } + } + + private fun readGroup(value: String, sink: PortabilityRecordSink) { + PortabilityJsonFactory.createParser(value).use { parser -> readGroupObject(parser, sink) } + } + + private fun readGroupObject(parser: JsonParser, sink: PortabilityRecordSink) { + parser.requireObject() + var name = "" + val urls = ArrayList() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "name" -> name = parser.textOrEmpty() + "urls" -> readUrlArray(parser, token, urls) + else -> parser.skipChildren() + } + } + if (name.isBlank()) return + sink.write(PortabilitySubscriptionGroup(name)) + urls.forEach { sink.write(PortabilitySubscriptionGroupMembership(name, it)) } + } + + private fun readUrlArray(parser: JsonParser, token: JsonToken, urls: MutableList) { + if (token != JsonToken.START_ARRAY) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(urls.size < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Grayjay group contains too many channels" } + if (parser.currentToken() == JsonToken.VALUE_STRING && parser.text.isNotBlank()) urls += parser.text + else parser.skipChildren() + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt new file mode 100644 index 00000000..da23f917 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream +import java.util.UUID +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +internal object GrayjayPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + ZipOutputStream(output.buffered()).use { zip -> + writeObjectEntry(zip, "exportInfo", mapOf("version" to "1")) + if (PortabilityCategory.SUBSCRIPTIONS in categories) { + writeStore(zip, "subscriptions") { json -> + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + (record as? PortabilitySubscription)?.let { json.writeString(it.channelUrl) } + } + } + } + if (PortabilityCategory.SUBSCRIPTION_GROUPS in categories) writeGroups(zip, source) + if (PortabilityCategory.HISTORY in categories) writeHistory(zip, source) + if (PortabilityCategory.PLAYLISTS in categories) writePlaylists(zip, source) + if (PortabilityCategory.WATCH_LATER in categories) writeWatchLater(zip, source) + writeObjectEntry(zip, "plugins", emptyMap()) + writeObjectEntry(zip, "plugin_settings", emptyMap()) + } + } + + private fun writeHistory(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "history") { json -> + source.forEach(PortabilityCategory.HISTORY) { record -> + val history = record as? PortabilityHistory ?: return@forEach + val title = history.video.title.sanitizedReconstructionText() + json.writeString("${history.video.url}|||${history.watchedAt}|||${history.positionSeconds}|||$title") + } + } + + private fun writeWatchLater(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "watch_later") { json -> + source.forEach(PortabilityCategory.WATCH_LATER) { record -> + (record as? PortabilityWatchLater)?.let { json.writeString(it.video.url) } + } + } + + private fun writePlaylists(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "playlists") { json -> + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + val playlist = record as? PortabilityPlaylist ?: return@forEach + val value = buildString { + append(playlist.name.sanitizedReconstructionText()) + append(":::") + append(playlist.sourceId.sanitizedReconstructionText()) + source.forEachChild(PortabilityCategory.PLAYLISTS, playlist.stableKey().removePrefix("playlist:")) { child -> + val item = child as? PortabilityPlaylistVideo ?: return@forEachChild + append('\n').append(item.video.url.replace("\n", "")) + } + } + json.writeString(value) + } + } + + private fun writeGroups(zip: ZipOutputStream, source: PortabilityRecordSource) = writeStore(zip, "subscription_groups") { json -> + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + val group = record as? PortabilitySubscriptionGroup ?: return@forEach + val value = buildGrayjayGroup(source, group) + require(value.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "Grayjay group is too large" } + json.writeString(value) + } + } + + private fun buildGrayjayGroup(source: PortabilityRecordSource, group: PortabilitySubscriptionGroup): String { + val bytes = java.io.ByteArrayOutputStream() + PortabilityJsonFactory.createGenerator(bytes).use { json -> + json.writeStartObject() + json.writeStringField("id", UUID.nameUUIDFromBytes(group.stableKey().toByteArray()).toString()) + json.writeStringField("name", group.name) + json.writeArrayFieldStart("urls") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, group.name) { child -> + (child as? PortabilitySubscriptionGroupMembership)?.let { json.writeString(it.channelUrl) } + } + json.writeEndArray() + json.writeNumberField("priority", 99) + json.writeEndObject() + } + return bytes.toString(Charsets.UTF_8) + } + + private fun writeObjectEntry(zip: ZipOutputStream, name: String, values: Map<*, *>) { + zip.putNextEntry(ZipEntry(name)) + val json = generator(zip) + json.writeStartObject() + values.forEach { (key, value) -> json.writeStringField(key.toString(), value.toString()) } + json.writeEndObject() + json.close() + zip.closeEntry() + } + + private inline fun writeStore(zip: ZipOutputStream, name: String, block: (JsonGenerator) -> Unit) { + zip.putNextEntry(ZipEntry("stores/$name")) + val json = generator(zip) + json.writeStartArray() + block(json) + json.writeEndArray() + json.close() + zip.closeEntry() + } + + private fun generator(output: OutputStream): JsonGenerator = PortabilityJsonFactory.createGenerator(output).apply { + disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) + } +} + +private fun String.sanitizedReconstructionText(): String = replace("|||", " ").replace(":::", " ").replace("\n", " ") diff --git a/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt new file mode 100644 index 00000000..dc3b9dcc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt @@ -0,0 +1,208 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonObject +import java.io.OutputStream + +class InvidiousPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.INVIDIOUS, + adapterVersion = 1, + capabilities = setOf( + capability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + capability(PortabilityCategory.HISTORY, PortabilityFidelity.PARTIAL), + capability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + capability(PortabilityCategory.SETTINGS, PortabilityFidelity.PARTIAL), + ), + defaultExtension = "json", + contentType = "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + val hasHistory = probe.contains("\"watch_history\"") + val hasPreferences = probe.contains("\"preferences\"") + val hasSubscriptions = probe.contains("\"subscriptions\"") + if (!hasSubscriptions || (!hasHistory && !hasPreferences)) return null + return PortabilityDetection(PortabilityFormat.INVIDIOUS, null, 96, "Invidious account export fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when { + field == "subscriptions" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readSubscriptions(parser, sink) + } + field == "watch_history" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.HISTORY) + readHistory(parser, sink) + } + field == "playlists" && token == JsonToken.START_ARRAY -> { + sink.markCategory(PortabilityCategory.PLAYLISTS) + readPlaylists(parser, sink) + } + field == "preferences" && token == JsonToken.START_OBJECT -> { + sink.markCategory(PortabilityCategory.SETTINGS) + sink.write(PortabilitySettings(JsonObject(mapOf("invidiousPreferences" to parser.readJsonElement())))) + } + else -> parser.skipChildren() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + writeSubscriptions(json, source, categories) + writeHistory(json, source, categories) + json.writeObjectFieldStart("preferences") + json.writeEndObject() + writePlaylists(json, source, categories) + json.writeEndObject() + } + } + + private fun readSubscriptions(parser: JsonParser, sink: PortabilityRecordSink) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + val channel = youtubeChannelUrl(parser.textOrEmpty()) + if (channel.isNotBlank()) sink.write(PortabilitySubscription(channel)) + } + } + + private fun readHistory(parser: JsonParser, sink: PortabilityRecordSink) { + var missingDates = 0L + while (parser.nextToken() != JsonToken.END_ARRAY) { + val url = youtubeVideoUrl(parser.textOrEmpty()) + if (url.isNotBlank()) { + sink.write(PortabilityHistory(PortabilityVideo(url), watchedAt = 0L)) + missingDates += 1 + } + } + if (missingDates > 0L) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "missing_history_dates", + "Invidious does not include original watch dates", + missingDates, + ), + ) + } + } + + private fun readPlaylists(parser: JsonParser, sink: PortabilityRecordSink) { + var index = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(parser.currentToken() == JsonToken.START_OBJECT) { "Invalid Invidious playlist" } + val playlist = readPlaylist(parser, index++) + sink.write(PortabilityPlaylist(playlist.id, playlist.title, playlist.description)) + playlist.videoIds.forEachIndexed { position, videoId -> + sink.write( + PortabilityPlaylistVideo( + playlist.id, + position, + PortabilityVideo(youtubeVideoUrl(videoId)), + ), + ) + } + } + } + + private fun readPlaylist(parser: JsonParser, index: Int): ParsedPlaylist { + var title = "" + var description = "" + val videoIds = mutableListOf() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when { + field == "title" -> title = parser.textOrEmpty() + field == "description" -> description = parser.textOrEmpty() + field == "videos" && token == JsonToken.START_ARRAY -> { + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(videoIds.size < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Playlist contains too many videos" } + videoIds += parser.textOrEmpty() + } + } + else -> parser.skipChildren() + } + } + val id = "invidious:$index:${title.trim().lowercase()}" + return ParsedPlaylist(id, title, description, videoIds) + } + + private fun writeSubscriptions( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("subscriptions") + if (PortabilityCategory.SUBSCRIPTIONS in categories) { + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + json.writeString(youtubeId((record as PortabilitySubscription).channelUrl)) + } + } + json.writeEndArray() + } + + private fun writeHistory( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("watch_history") + if (PortabilityCategory.HISTORY in categories) { + source.forEach(PortabilityCategory.HISTORY) { record -> + json.writeString(youtubeId((record as PortabilityHistory).video.url)) + } + } + json.writeEndArray() + } + + private fun writePlaylists( + json: com.fasterxml.jackson.core.JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + json.writeArrayFieldStart("playlists") + if (PortabilityCategory.PLAYLISTS in categories) { + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("title", record.name) + json.writeStringField("description", record.description) + json.writeStringField("privacy", "Private") + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + if (child is PortabilityPlaylistVideo) json.writeString(youtubeId(child.video.url)) + } + json.writeEndArray() + json.writeEndObject() + } + } + json.writeEndArray() + } + + private data class ParsedPlaylist( + val id: String, + val title: String, + val description: String, + val videoIds: List, + ) +} + +private fun capability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt new file mode 100644 index 00000000..d755c94a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt @@ -0,0 +1,76 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class LibreTubePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.LIBRE_TUBE, + 1, + setOf( + libreCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.HISTORY, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.PROGRESS, PortabilityFidelity.COMPLETE), + libreCapability(PortabilityCategory.SEARCH_HISTORY, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + libreCapability(PortabilityCategory.SAVED_PLAYLISTS, PortabilityFidelity.COMPLETE), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (!probe.contains("\"format\"") || !probe.contains("\"Piped\"")) return null + val evidence = UNIQUE_FIELDS.firstOrNull(probe::contains) ?: return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.LIBRE_TUBE, version, 99, "LibreTube field $evidence") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions", "localSubscriptions" -> PipedPortabilityReader.subscriptions(parser, sink, token) + "groups", "channelGroups" -> PipedPortabilityReader.groups(parser, sink, token) + "watchHistory" -> LibreTubePortabilityReader.history(parser, sink, token) + "watchPositions" -> LibreTubePortabilityReader.positions(parser, sink, token) + "searchHistory" -> LibreTubePortabilityReader.searchHistory(parser, sink, token) + "localPlaylists" -> LibreTubePortabilityReader.localPlaylists(parser, sink, token) + "playlistBookmarks" -> LibreTubePortabilityReader.bookmarks(parser, sink, token) + else -> parser.skipChildren() + } + } + require(format == "Piped" && version == 1) { "Unsupported LibreTube backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = LibreTubePortabilityWriter.write(source, output, categories) + + private companion object { + val UNIQUE_FIELDS = listOf( + "\"watchPositions\"", + "\"customInstances\"", + "\"playlistBookmarks\"", + "\"localPlaylists\"", + "\"preferences\"", + ) + } +} + +private fun libreCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt new file mode 100644 index 00000000..afb7c4f2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt @@ -0,0 +1,106 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonObject + +internal object LibreTubePortabilityReader { + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.HISTORY, parser, token) { element -> + val item = element.jsonObject + val id = item.string("videoId") + if (id.isBlank()) return@readArray + sink.write( + PortabilityHistory( + PortabilityVideo( + youtubeVideoUrl(id), + item.string("title"), + item.string("thumbnailUrl"), + item.long("duration"), + item.string("uploader"), + item.string("uploaderUrl"), + item.string("uploaderAvatar"), + ), + 0L, + ), + ) + } + } + + fun positions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.PROGRESS, parser, token) { element -> + val item = element.jsonObject + val id = item.string("videoId") + if (id.isNotBlank()) sink.write(PortabilityProgress(youtubeVideoUrl(id), item.long("position") / 1_000L)) + } + } + + fun searchHistory(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.SEARCH_HISTORY, parser, token) { element -> + val query = element.jsonObject.string("query") + if (query.isNotBlank()) sink.write(PortabilitySearchHistory(query, 0L)) + } + } + + fun localPlaylists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.PLAYLISTS, parser, token) { element -> + val item = element.jsonObject + val playlist = item["playlist"]?.objectOrNull() ?: return@readArray + val sourceId = "libretube:${playlist.string("id")}:${playlist.string("name").lowercase()}" + sink.write(PortabilityPlaylist(sourceId, playlist.string("name"), playlist.string("description"))) + item.array("videos").forEachIndexed { position, videoElement -> + val video = videoElement.jsonObject + sink.write( + PortabilityPlaylistVideo( + sourceId, + position, + PortabilityVideo( + youtubeVideoUrl(video.string("videoId")), + video.string("title"), + video.string("thumbnailUrl"), + video.long("duration"), + video.string("uploader"), + video.string("uploaderUrl"), + video.string("uploaderAvatar"), + ), + ), + ) + } + } + } + + fun bookmarks(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + sink.readArray(PortabilityCategory.SAVED_PLAYLISTS, parser, token) { element -> + val item = element.jsonObject + val id = item.string("playlistId") + if (id.isNotBlank()) { + sink.write( + PortabilitySavedPlaylist( + id, + "https://www.youtube.com/playlist?list=$id", + item.string("playlistName"), + item.string("thumbnailUrl"), + item.string("uploader"), + item.long("videos"), + ), + ) + } + } + } +} + +private fun PortabilityRecordSink.readArray( + category: PortabilityCategory, + parser: JsonParser, + token: JsonToken, + block: (JsonElement) -> Unit, +) { + require(token == JsonToken.START_ARRAY) { "Invalid LibreTube backup section" } + markCategory(category) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement()) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt new file mode 100644 index 00000000..f3eb1915 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt @@ -0,0 +1,145 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object LibreTubePortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", "Piped") + json.writeNumberField("version", 1) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + positions(json, source, categories) + searchHistory(json, source, categories) + localPlaylists(json, source, categories) + bookmarks(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("localSubscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + json.writeStartObject() + json.writeStringField("channelId", youtubeId(item.channelUrl)) + json.writeStringField("name", item.name) + json.writeStringField("avatar", item.avatarUrl) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("channelGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("groupName", record.name) + json.writeArrayFieldStart("channels") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + json.writeString(youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl)) + } + json.writeEndArray() + json.writeNumberField("index", 0) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("watchHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("uploader", item.video.channelName) + json.writeStringField("uploaderUrl", item.video.channelUrl) + json.writeStringField("uploaderAvatar", item.video.channelAvatarUrl) + json.writeStringField("thumbnailUrl", item.video.thumbnailUrl) + json.writeNumberField("duration", item.video.durationSeconds) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun positions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PROGRESS !in categories) return + json.writeArrayFieldStart("watchPositions") + source.forEach(PortabilityCategory.PROGRESS) { record -> + val item = record as PortabilityProgress + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.videoUrl)) + json.writeNumberField("position", item.positionSeconds * 1_000L) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + json.writeStartObject() + json.writeStringField("query", (record as PortabilitySearchHistory).term) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun localPlaylists(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("localPlaylists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeObjectFieldStart("playlist") + json.writeNumberField("id", 0) + json.writeStringField("name", record.name) + json.writeStringField("description", record.description) + json.writeStringField("thumbnailUrl", "") + json.writeEndObject() + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + val video = (child as PortabilityPlaylistVideo).video + json.writeStartObject() + json.writeNumberField("id", 0) + json.writeNumberField("playlistId", 0) + json.writeStringField("videoId", youtubeId(video.url)) + json.writeStringField("title", video.title) + json.writeStringField("thumbnailUrl", video.thumbnailUrl) + json.writeNumberField("duration", video.durationSeconds) + json.writeStringField("uploader", video.channelName) + json.writeStringField("uploaderUrl", video.channelUrl) + json.writeStringField("uploaderAvatar", video.channelAvatarUrl) + json.writeEndObject() + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } + + private fun bookmarks(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SAVED_PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlistBookmarks") + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + val item = record as PortabilitySavedPlaylist + json.writeStartObject() + json.writeStringField("playlistId", item.sourceId) + json.writeStringField("playlistName", item.title) + json.writeStringField("thumbnailUrl", item.thumbnailUrl) + json.writeStringField("uploader", item.uploaderName) + json.writeNumberField("videos", item.streamCount) + json.writeEndObject() + } + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt new file mode 100644 index 00000000..a7a7db4c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt @@ -0,0 +1,75 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class MaterialiousPortabilityAdapter : PortabilityAdapter { + private val opml = OpmlPortabilityAdapter(PortabilityFormat.MATERIALIOUS, autoDetect = false) + + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.MATERIALIOUS, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.PARTIAL, + ), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + if (opml.detect(input) != null) { + return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "opml", 82, "Materialious-compatible OPML") + } + val probe = input.probe.decodeToString() + if (!probe.trimStart().startsWith("{") || !probe.contains("\"subscriptions\"")) return null + if (probe.contains("\"app_version")) return null + val confidence = if (input.filename.contains("materialious", ignoreCase = true)) 99 else 86 + return PortabilityDetection(PortabilityFormat.MATERIALIOUS, "invidious-subscriptions", confidence, "Materialious Invidious subscription JSON") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + val detection = requireNotNull(detect(input)) { "Unsupported Materialious export" } + if (detection.formatVersion == "opml") return opml.decode(input, sink) + input.withJsonParser { parser -> + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + if (field == "subscriptions" && token == JsonToken.START_ARRAY) readSubscriptions(parser, sink) + else parser.skipChildren() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + require(PortabilityCategory.SUBSCRIPTIONS in categories) { "Materialious export requires subscriptions" } + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + (record as? PortabilitySubscription)?.let { json.writeString(youtubeId(it.channelUrl)) } + } + json.writeEndArray() + json.writeEndObject() + } + } + + private fun readSubscriptions(parser: com.fasterxml.jackson.core.JsonParser, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "Materialious export contains too many subscriptions" } + val channelUrl = youtubeChannelUrl(parser.textOrEmpty()) + if (channelUrl.isNotBlank()) sink.write(PortabilitySubscription(channelUrl)) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt new file mode 100644 index 00000000..3c99df75 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +internal fun newPipeArchiveCapabilities(): Set = + archiveImportCapabilities().mapTo(linkedSetOf()) { capability -> + PortabilityCapability( + capability.category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + when (capability.category) { + PortabilityCategory.PLAYLISTS -> PortabilityFidelity.PARTIAL + else -> PortabilityFidelity.COMPLETE + }, + ) + } + +internal fun newPipeArchiveExportIssues( + source: PortabilityRecordSource, + categories: Set, + target: NewPipeArchiveTarget, +): List { + val counts = linkedMapOf() + categories.forEach { category -> + source.forEach(category) { record -> + record.urls().forEach { url -> + if (!NewPipeProvider.supported(url, target)) counts[category] = (counts[category] ?: 0L) + 1L + } + } + } + return counts.map { (category, count) -> + PortabilityIssue( + category, + "unsupported_provider", + "${target.name.replace('_', ' ')} cannot represent one or more provider records", + count, + ) + } +} + +private fun PortabilityRecord.urls(): List = when (this) { + is PortabilitySubscription -> listOf(channelUrl) + is PortabilitySubscriptionGroupMembership -> listOf(channelUrl) + is PortabilityHistory -> listOf(video.url) + is PortabilityPlaylistVideo -> listOf(video.url) + is PortabilityProgress -> listOf(videoUrl) + is PortabilitySavedPlaylist -> listOf(url) + else -> emptyList() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt new file mode 100644 index 00000000..ad822179 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt @@ -0,0 +1,63 @@ +package dev.typetype.server.portability + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.file.Files +import java.sql.Connection +import java.sql.DriverManager +import java.util.zip.ZipFile + +internal object NewPipeArchiveDatabase { + private val databaseNames = setOf("newpipe.db", "pipepipe.db") + + fun userVersion(input: PortabilityInput): Int? { + val entryName = databaseEntry(input) ?: return null + return ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry(entryName)) + val header = zip.getInputStream(entry).use { it.readNBytes(SQLITE_HEADER_BYTES) } + if (header.size < SQLITE_HEADER_BYTES || !header.startsWith(SQLITE_SIGNATURE)) return null + ByteBuffer.wrap(header, USER_VERSION_OFFSET, Int.SIZE_BYTES) + .order(ByteOrder.BIG_ENDIAN) + .int + } + } + + fun read(input: PortabilityInput, block: (Connection) -> T): T { + val entryName = requireNotNull(databaseEntry(input)) { "Backup database is missing" } + val extracted = Files.createTempFile(requireNotNull(input.path.parent), "portability-newpipe-", ".sqlite") + try { + ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry(entryName)) + zip.getInputStream(entry).use { source -> + Files.newOutputStream(extracted).use { target -> source.copyTo(target) } + } + } + require(Files.size(extracted) in 1..PortabilityLimits.MAX_ARCHIVE_ENTRY_BYTES) { + "Backup database is outside the allowed range" + } + Class.forName("org.sqlite.JDBC") + return DriverManager.getConnection("jdbc:sqlite:file:${extracted.toAbsolutePath()}?mode=ro&immutable=1").use { connection -> + connection.createStatement().use { statement -> + statement.execute("PRAGMA query_only = ON") + statement.execute("PRAGMA trusted_schema = OFF") + statement.execute("PRAGMA cell_size_check = ON") + } + block(connection) + } + } finally { + Files.deleteIfExists(extracted) + } + } + + private fun databaseEntry(input: PortabilityInput): String? = input.archive?.entries + ?.map(PortabilityArchiveEntry::name) + ?.filter { it.substringAfterLast('/').lowercase() in databaseNames } + ?.singleOrNull() + + private fun ByteArray.startsWith(prefix: ByteArray): Boolean = + size >= prefix.size && prefix.indices.all { this[it] == prefix[it] } + + private const val SQLITE_HEADER_BYTES = 64 + private const val USER_VERSION_OFFSET = 60 + private val SQLITE_SIGNATURE = "SQLite format 3\u0000".toByteArray() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt new file mode 100644 index 00000000..96d2b206 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.nio.file.Files +import java.sql.DriverManager +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +internal object NewPipeArchivePortabilityWriter { + fun write( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + target: NewPipeArchiveTarget, + ) { + val database = Files.createTempFile("typetype-portability-", ".db") + try { + DriverManager.getConnection("jdbc:sqlite:$database").use { db -> + db.autoCommit = false + NewPipeArchiveSchema.create(db, target) + NewPipeArchiveRecordWriter(db, target).write(source, categories) + db.commit() + } + ZipOutputStream(output).use { zip -> + zip.putNextEntry(ZipEntry("newpipe.db")) + Files.newInputStream(database).use { it.copyTo(zip) } + zip.closeEntry() + } + } finally { + Files.deleteIfExists(database) + Files.deleteIfExists(database.resolveSibling("${database.fileName}-journal")) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt new file mode 100644 index 00000000..8c08762a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt @@ -0,0 +1,179 @@ +package dev.typetype.server.portability + +import java.sql.Connection + +internal class NewPipeArchiveRecordWriter( + private val db: Connection, + private val target: NewPipeArchiveTarget, +) { + fun write(source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS in categories) subscriptions(source) + if (PortabilityCategory.SUBSCRIPTION_GROUPS in categories) groups(source) + if (PortabilityCategory.HISTORY in categories) history(source) + if (PortabilityCategory.PLAYLISTS in categories) playlists(source) + if (PortabilityCategory.PROGRESS in categories) progress(source) + if (PortabilityCategory.SEARCH_HISTORY in categories) searchHistory(source) + if (PortabilityCategory.SAVED_PLAYLISTS in categories) savedPlaylists(source) + } + + private fun subscriptions(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + if (NewPipeProvider.supported(item.channelUrl, target)) ensureSubscription(item.channelUrl, item.name, item.avatarUrl) + } + } + + private fun groups(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val groupId = db.insertId( + "INSERT INTO feed_group(name,icon_id,sort_order) VALUES(?,?,?)", + record.name, + 0, + order++, + ) + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + val membership = child as? PortabilitySubscriptionGroupMembership ?: return@forEachChild + if (!NewPipeProvider.supported(membership.channelUrl, target)) return@forEachChild + val subscriptionId = ensureSubscription(membership.channelUrl, "", "") + db.execute( + "INSERT OR IGNORE INTO feed_group_subscription_join(group_id,subscription_id) VALUES(?,?)", + groupId, + subscriptionId, + ) + } + } + } + + private fun history(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + if (!NewPipeProvider.supported(item.video.url, target)) return@forEach + val streamId = ensureVideo(item.video) + db.execute( + "INSERT OR IGNORE INTO stream_history(stream_id,access_date,repeat_count) VALUES(?,?,1)", + streamId, + item.watchedAt, + ) + } + } + + private fun playlists(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + val playlist = record as? PortabilityPlaylist ?: return@forEach + val id = if (target.pipePipe) { + db.insertId("INSERT INTO playlists(name,thumbnail_url,display_index) VALUES(?,'',?)", playlist.name, order++) + } else { + db.insertId( + "INSERT INTO playlists(name,is_thumbnail_permanent,thumbnail_stream_id,display_index) VALUES(?,0,-1,?)", + playlist.name, + order++, + ) + } + source.forEachChild(PortabilityCategory.PLAYLISTS, playlist.sourceId) { child -> + val item = child as? PortabilityPlaylistVideo ?: return@forEachChild + if (!NewPipeProvider.supported(item.video.url, target)) return@forEachChild + db.execute( + "INSERT OR REPLACE INTO playlist_stream_join(playlist_id,stream_id,join_index) VALUES(?,?,?)", + id, + ensureVideo(item.video), + item.position, + ) + } + } + } + + private fun progress(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.PROGRESS) { record -> + val item = record as PortabilityProgress + if (!NewPipeProvider.supported(item.videoUrl, target)) return@forEach + val streamId = ensureVideo(PortabilityVideo(item.videoUrl)) + db.execute("INSERT OR REPLACE INTO stream_state(stream_id,progress_time) VALUES(?,?)", streamId, item.positionSeconds) + } + } + + private fun searchHistory(source: PortabilityRecordSource) { + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + val item = record as PortabilitySearchHistory + db.execute( + "INSERT INTO search_history(creation_date,service_id,search) VALUES(?,0,?)", + item.searchedAt, + item.term, + ) + } + } + + private fun savedPlaylists(source: PortabilityRecordSource) { + var order = 0 + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + val item = record as PortabilitySavedPlaylist + val serviceId = NewPipeProvider.serviceId(item.url) ?: return@forEach + if (!target.pipePipe && serviceId != 0) return@forEach + db.execute( + "INSERT OR IGNORE INTO remote_playlists(service_id,name,url,thumbnail_url,uploader,display_index,stream_count) VALUES(?,?,?,?,?,?,?)", + serviceId, + item.title, + item.url, + item.thumbnailUrl, + item.uploaderName, + order++, + item.streamCount, + ) + } + } + + private fun ensureSubscription(url: String, name: String, avatar: String): Long { + val serviceId = requireNotNull(NewPipeProvider.serviceId(url)) + db.execute( + "INSERT OR IGNORE INTO subscriptions(service_id,url,name,avatar_url,subscriber_count,description,notification_mode) VALUES(?,?,?,?,NULL,NULL,0)", + serviceId, + url, + name, + avatar, + ) + return db.long("SELECT uid FROM subscriptions WHERE service_id=? AND url=?", serviceId, url) + } + + private fun ensureVideo(video: PortabilityVideo): Long { + val serviceId = requireNotNull(NewPipeProvider.serviceId(video.url)) + val columns = if (target.pipePipe) { + "service_id,url,title,stream_type,duration,uploader,uploader_url,thumbnail_url,view_count,textual_upload_date,upload_date,is_upload_date_approximation,is_paid" + } else { + "service_id,url,title,stream_type,duration,uploader,uploader_url,thumbnail_url,view_count,textual_upload_date,upload_date,is_upload_date_approximation" + } + val values = if (target.pipePipe) "?,?,?,?,?,?,?,?,?,NULL,?,0,0" else "?,?,?,?,?,?,?,?,?,NULL,?,0" + db.execute( + "INSERT OR IGNORE INTO streams($columns) VALUES($values)", + serviceId, + video.url, + video.title, + "VIDEO_STREAM", + video.durationSeconds, + video.channelName, + video.channelUrl, + video.thumbnailUrl, + video.viewCount, + video.publishedAt.takeIf { it >= 0L }, + ) + return db.long("SELECT uid FROM streams WHERE service_id=? AND url=?", serviceId, video.url) + } +} + +private fun Connection.execute(sql: String, vararg values: Any?) { + prepareStatement(sql).use { statement -> + values.forEachIndexed { index, value -> statement.setObject(index + 1, value) } + statement.executeUpdate() + } +} + +private fun Connection.insertId(sql: String, vararg values: Any?): Long { + execute(sql, *values) + return long("SELECT last_insert_rowid()") +} + +private fun Connection.long(sql: String, vararg values: Any?): Long = prepareStatement(sql).use { statement -> + values.forEachIndexed { index, value -> statement.setObject(index + 1, value) } + statement.executeQuery().use { rows -> check(rows.next()); rows.getLong(1) } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt new file mode 100644 index 00000000..e83b543f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import java.sql.Connection + +internal object NewPipeArchiveSchema { + fun create(db: Connection, target: NewPipeArchiveTarget) { + db.createStatement().use { statement -> + statement.execute("PRAGMA foreign_keys = ON") + statement.execute("PRAGMA journal_mode = DELETE") + statements(target).forEach(statement::execute) + statement.execute("PRAGMA user_version = ${target.databaseVersion}") + statement.execute("CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)") + statement.execute("INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42,'${target.identityHash}')") + } + db.commit() + } + + private fun statements(target: NewPipeArchiveTarget): List = listOf( + "CREATE TABLE subscriptions (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT, name TEXT, avatar_url TEXT, subscriber_count INTEGER, description TEXT, notification_mode INTEGER NOT NULL)", + "CREATE UNIQUE INDEX index_subscriptions_service_id_url ON subscriptions(service_id,url)", + "CREATE TABLE search_history (creation_date INTEGER, service_id INTEGER NOT NULL, search TEXT, id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL)", + "CREATE INDEX index_search_history_search ON search_history(search)", + streams(target), + "CREATE UNIQUE INDEX index_streams_service_id_url ON streams(service_id,url)", + "CREATE TABLE stream_history (stream_id INTEGER NOT NULL, access_date INTEGER NOT NULL, repeat_count INTEGER NOT NULL, PRIMARY KEY(stream_id,access_date), FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE)", + "CREATE INDEX index_stream_history_stream_id ON stream_history(stream_id)", + "CREATE TABLE stream_state (stream_id INTEGER NOT NULL PRIMARY KEY, progress_time INTEGER NOT NULL, FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE)", + playlists(target), + "CREATE TABLE playlist_stream_join (playlist_id INTEGER NOT NULL, stream_id INTEGER NOT NULL, join_index INTEGER NOT NULL, PRIMARY KEY(playlist_id,join_index), FOREIGN KEY(playlist_id) REFERENCES playlists(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE UNIQUE INDEX index_playlist_stream_join_playlist_id_join_index ON playlist_stream_join(playlist_id,join_index)", + "CREATE INDEX index_playlist_stream_join_stream_id ON playlist_stream_join(stream_id)", + "CREATE TABLE remote_playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, name TEXT, url TEXT, thumbnail_url TEXT, uploader TEXT, display_index INTEGER NOT NULL, stream_count INTEGER)", + "CREATE UNIQUE INDEX index_remote_playlists_service_id_url ON remote_playlists(service_id,url)", + "CREATE TABLE feed (stream_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(stream_id,subscription_id), FOREIGN KEY(stream_id) REFERENCES streams(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE INDEX index_feed_subscription_id ON feed(subscription_id)", + "CREATE TABLE feed_group (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, icon_id INTEGER NOT NULL, sort_order INTEGER NOT NULL)", + "CREATE INDEX index_feed_group_sort_order ON feed_group(sort_order)", + "CREATE TABLE feed_group_subscription_join (group_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(group_id,subscription_id), FOREIGN KEY(group_id) REFERENCES feed_group(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "CREATE INDEX index_feed_group_subscription_join_subscription_id ON feed_group_subscription_join(subscription_id)", + "CREATE TABLE feed_last_updated (subscription_id INTEGER NOT NULL PRIMARY KEY, last_updated INTEGER, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + ) + targetIndexes(target) + + private fun streams(target: NewPipeArchiveTarget): String { + val paid = if (target.pipePipe) ", is_paid INTEGER NOT NULL" else "" + return "CREATE TABLE streams (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, stream_type TEXT NOT NULL, duration INTEGER NOT NULL, uploader TEXT NOT NULL, uploader_url TEXT, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER, is_upload_date_approximation INTEGER$paid)" + } + + private fun playlists(target: NewPipeArchiveTarget): String = if (target.pipePipe) { + "CREATE TABLE playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT, thumbnail_url TEXT, display_index INTEGER NOT NULL)" + } else { + "CREATE TABLE playlists (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT, is_thumbnail_permanent INTEGER NOT NULL, thumbnail_stream_id INTEGER NOT NULL, display_index INTEGER NOT NULL)" + } + + private fun targetIndexes(target: NewPipeArchiveTarget): List = if (target.pipePipe) { + listOf( + "CREATE INDEX index_playlists_name ON playlists(name)", + "CREATE INDEX index_remote_playlists_name ON remote_playlists(name)", + ) + } else { + emptyList() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt new file mode 100644 index 00000000..61bf8305 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.portability + +internal enum class NewPipeArchiveTarget( + val databaseVersion: Int, + val identityHash: String, + val pipePipe: Boolean, +) { + NEW_PIPE(9, "7591e8039faa74d8c0517dc867af9d3e", false), + PIPE_PIPE(901, "d505dd6c0be6a80da07aa980bd361064", true), +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt new file mode 100644 index 00000000..87d057f8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt @@ -0,0 +1,151 @@ +package dev.typetype.server.portability + +import java.sql.Connection +import java.sql.ResultSet + +internal object NewPipeDatabasePortabilityReader { + fun read(connection: Connection, sink: PortabilityRecordSink) { + subscriptions(connection, sink) + groups(connection, sink) + history(connection, sink) + playlists(connection, sink) + progress(connection, sink) + searchHistory(connection, sink) + } + + private fun subscriptions(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.SUBSCRIPTIONS, + "subscriptions", + "SELECT service_id, url, name, avatar_url FROM subscriptions ORDER BY uid", + ) { row -> + if (row.int("service_id") == 0) { + sink.write(PortabilitySubscription(row.string("url"), row.string("name"), row.string("avatar_url"))) + } else { + sink.issue(PortabilityIssue(PortabilityCategory.SUBSCRIPTIONS, "unsupported_subscription_provider", "A non-YouTube subscription was skipped")) + } + } + + private fun groups(db: Connection, sink: PortabilityRecordSink) { + sink.readTable(db, PortabilityCategory.SUBSCRIPTION_GROUPS, "feed_group", "SELECT uid, name FROM feed_group ORDER BY sort_order, uid") { row -> + sink.write(PortabilitySubscriptionGroup(row.string("name"))) + } + if (!db.hasTables("feed_group_subscription_join", "feed_group", "subscriptions")) return + db.each( + """ + SELECT g.name AS group_name, s.url AS channel_url + FROM feed_group_subscription_join j + JOIN feed_group g ON g.uid = j.group_id + JOIN subscriptions s ON s.uid = j.subscription_id + ORDER BY j.group_id, j.subscription_id + """.trimIndent(), + ) { row -> + sink.write(PortabilitySubscriptionGroupMembership(row.string("group_name"), row.string("channel_url"))) + } + } + + private fun history(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.HISTORY, + "stream_history", + """ + SELECT h.access_date, s.url, s.title, s.duration, s.uploader, s.uploader_url, s.thumbnail_url + FROM stream_history h JOIN streams s ON s.uid = h.stream_id + ORDER BY h.access_date, h.stream_id + """.trimIndent(), + requiredTables = arrayOf("streams"), + ) { row -> + sink.write( + PortabilityHistory( + row.video(), + row.long("access_date"), + ), + ) + } + + private fun playlists(db: Connection, sink: PortabilityRecordSink) { + sink.readTable(db, PortabilityCategory.PLAYLISTS, "playlists", "SELECT uid, name FROM playlists ORDER BY display_index, uid") { row -> + sink.write(PortabilityPlaylist(row.long("uid").toString(), row.string("name"))) + } + if (db.hasTables("playlist_stream_join", "streams")) { + db.each( + """ + SELECT j.playlist_id, j.join_index, s.url, s.title, s.duration, s.uploader, s.uploader_url, s.thumbnail_url + FROM playlist_stream_join j JOIN streams s ON s.uid = j.stream_id + ORDER BY j.playlist_id, j.join_index + """.trimIndent(), + ) { row -> + sink.write(PortabilityPlaylistVideo(row.long("playlist_id").toString(), row.int("join_index"), row.video())) + } + } + sink.readTable( + db, + PortabilityCategory.SAVED_PLAYLISTS, + "remote_playlists", + "SELECT uid, url, name, thumbnail_url, uploader, stream_count FROM remote_playlists ORDER BY display_index, uid", + ) { row -> + sink.write( + PortabilitySavedPlaylist( + row.long("uid").toString(), row.string("url"), row.string("name"), + row.string("thumbnail_url"), row.string("uploader"), row.long("stream_count"), + ), + ) + } + } + + private fun progress(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.PROGRESS, + "stream_state", + "SELECT s.url, state.progress_time FROM stream_state state JOIN streams s ON s.uid = state.stream_id", + requiredTables = arrayOf("streams"), + ) { row -> + sink.write(PortabilityProgress(row.string("url"), row.long("progress_time"))) + } + + private fun searchHistory(db: Connection, sink: PortabilityRecordSink) = sink.readTable( + db, + PortabilityCategory.SEARCH_HISTORY, + "search_history", + "SELECT search, creation_date FROM search_history ORDER BY creation_date, id", + ) { row -> + sink.write(PortabilitySearchHistory(row.string("search"), row.long("creation_date"))) + } +} + +private fun PortabilityRecordSink.readTable( + db: Connection, + category: PortabilityCategory, + table: String, + sql: String, + requiredTables: Array = emptyArray(), + block: (SqliteRow) -> Unit, +) { + if (!db.hasTables(table, *requiredTables)) return + markCategory(category) + db.each(sql, block) +} + +private fun Connection.each(sql: String, block: (SqliteRow) -> Unit) { + prepareStatement(sql).use { statement -> + statement.fetchSize = 256 + statement.executeQuery().use { rows -> while (rows.next()) block(SqliteRow(rows)) } + } +} + +private fun Connection.hasTables(vararg names: String): Boolean = names.all { name -> + prepareStatement("SELECT 1 FROM sqlite_master WHERE type='table' AND lower(name)=lower(?)").use { statement -> + statement.setString(1, name) + statement.executeQuery().use(ResultSet::next) + } +} + +private class SqliteRow(private val row: ResultSet) { + fun string(name: String): String = row.getString(name) ?: "" + fun int(name: String): Int = row.getInt(name) + fun long(name: String): Long = row.getLong(name) + fun video() = PortabilityVideo( + string("url"), string("title"), string("thumbnail_url"), long("duration"), + string("uploader"), string("uploader_url"), + ) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt new file mode 100644 index 00000000..12f228aa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class NewPipePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.NEW_PIPE, + adapterVersion = 3, + capabilities = newPipeArchiveCapabilities(), + defaultExtension = "zip", + contentType = "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) { + val version = NewPipeArchiveDatabase.userVersion(input) ?: return null + if (version !in 1 until PIPE_PIPE_DATABASE_VERSION) return null + return PortabilityDetection(PortabilityFormat.NEW_PIPE, version.toString(), 100, "NewPipe SQLite schema version") + } + val probe = input.probe.decodeToString() + if (!probe.contains("\"subscriptions\"") || !probe.contains("\"app_version")) return null + return PortabilityDetection(PortabilityFormat.NEW_PIPE, null, 98, "NewPipe subscription fields") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + if (input.archive != null) { + requireNotNull(detect(input)) { "Unsupported NewPipe backup" } + NewPipeArchiveDatabase.read(input) { NewPipeDatabasePortabilityReader.read(it, sink) } + return + } + decodeSubscriptions(input, sink) + } + + private fun decodeSubscriptions(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var foundSubscriptions = false + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val valueToken = parser.nextToken() + if (field == "subscriptions" && valueToken == JsonToken.START_ARRAY) { + foundSubscriptions = true + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readSubscriptions(parser, sink) + } else { + parser.skipChildren() + } + } + require(foundSubscriptions) { "NewPipe backup does not contain subscriptions" } + } + + private companion object { + const val PIPE_PIPE_DATABASE_VERSION = 900 + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List { + return super.assessExport(source, categories) + + newPipeArchiveExportIssues(source, categories, NewPipeArchiveTarget.NEW_PIPE) + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + NewPipeArchivePortabilityWriter.write(source, output, categories, NewPipeArchiveTarget.NEW_PIPE) + } + + private fun readSubscriptions( + parser: com.fasterxml.jackson.core.JsonParser, + sink: PortabilityRecordSink, + ) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(parser.currentToken() == JsonToken.START_OBJECT) { "Invalid NewPipe subscription" } + var serviceId = -1 + var url = "" + var name = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "service_id" -> serviceId = parser.intValue + "url" -> url = parser.textOrEmpty() + "name" -> name = parser.textOrEmpty() + else -> parser.skipChildren() + } + } + if (serviceId != 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.SUBSCRIPTIONS, + "unsupported_subscription_provider", + "A non-YouTube NewPipe subscription was skipped", + ), + ) + } else if (url.isNotBlank()) { + sink.write(PortabilitySubscription(url.trim(), name.trim())) + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt b/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt new file mode 100644 index 00000000..084cf6bc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.portability + +import java.net.URI + +internal object NewPipeProvider { + fun serviceId(url: String): Int? { + val host = runCatching { URI(url.trim()).host?.lowercase() }.getOrNull() ?: return null + return when { + host == "youtube.com" || host == "youtu.be" || host.endsWith(".youtube.com") -> 0 + host == "bilibili.com" || host.endsWith(".bilibili.com") || host == "b23.tv" -> 5 + host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") || host == "nico.ms" -> 6 + else -> null + } + } + + fun supported(url: String, target: NewPipeArchiveTarget): Boolean { + val id = serviceId(url) ?: return false + return target.pipePipe || id == 0 + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt new file mode 100644 index 00000000..5b1438b5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt @@ -0,0 +1,116 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import javax.xml.stream.XMLInputFactory +import javax.xml.stream.XMLOutputFactory +import javax.xml.stream.XMLStreamConstants + +class OpmlPortabilityAdapter( + private val format: PortabilityFormat = PortabilityFormat.OPML, + override val autoDetect: Boolean = format == PortabilityFormat.OPML, +) : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.PARTIAL, + ), + ), + "opml", + "application/xml", + ) + + init { + require(format in COMPATIBLE_FORMATS) { "Format does not use OPML" } + } + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString().trimStart() + if (!probe.startsWith("<") || !probe.contains("]*version=[\"']([^\"']+)", RegexOption.IGNORE_CASE) + .find(probe)?.groupValues?.get(1) + return PortabilityDetection(format, version, 92, "OPML document root") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + val factory = secureInputFactory() + java.nio.file.Files.newInputStream(input.path).buffered().use { stream -> + val reader = factory.createXMLStreamReader(stream) + var outlines = 0 + try { + while (reader.hasNext()) { + if (reader.next() != XMLStreamConstants.START_ELEMENT || reader.localName != "outline") continue + require(outlines++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "OPML contains too many outlines" } + val rawUrl = reader.attribute("xmlUrl").ifBlank { reader.attribute("url") } + val channelUrl = normalizeOpmlChannel(rawUrl) + if (channelUrl.isNotBlank()) { + val name = reader.attribute("title").ifBlank { reader.attribute("text") } + sink.write(PortabilitySubscription(channelUrl, name)) + } + } + } finally { + reader.close() + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + require(PortabilityCategory.SUBSCRIPTIONS in categories) { "OPML export requires subscriptions" } + val writer = XMLOutputFactory.newFactory().createXMLStreamWriter(output, Charsets.UTF_8.name()) + writer.writeStartDocument(Charsets.UTF_8.name(), "1.0") + writer.writeStartElement("opml") + writer.writeAttribute("version", "2.0") + writer.writeStartElement("head") + writer.writeStartElement("title") + writer.writeCharacters("TypeType subscriptions") + writer.writeEndElement() + writer.writeEndElement() + writer.writeStartElement("body") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + val item = record as PortabilitySubscription + val channelId = youtubeId(item.channelUrl) + writer.writeEmptyElement("outline") + writer.writeAttribute("text", item.name.ifBlank { channelId }) + writer.writeAttribute("title", item.name.ifBlank { channelId }) + writer.writeAttribute("type", "rss") + writer.writeAttribute("xmlUrl", "https://www.youtube.com/feeds/videos.xml?channel_id=$channelId") + writer.writeAttribute("htmlUrl", item.channelUrl) + } + writer.writeEndElement() + writer.writeEndElement() + writer.writeEndDocument() + writer.close() + } + + private companion object { + val COMPATIBLE_FORMATS = setOf( + PortabilityFormat.OPML, + PortabilityFormat.MATERIALIOUS, + PortabilityFormat.SKY_TUBE, + PortabilityFormat.YOUTUBE_LOCAL, + ) + } +} + +private fun secureInputFactory(): XMLInputFactory = XMLInputFactory.newFactory().apply { + setProperty(XMLInputFactory.SUPPORT_DTD, false) + setProperty("javax.xml.stream.isSupportingExternalEntities", false) +} + +private fun javax.xml.stream.XMLStreamReader.attribute(name: String): String = + (0 until attributeCount).firstOrNull { getAttributeLocalName(it) == name }?.let(::getAttributeValue).orEmpty() + +private fun normalizeOpmlChannel(value: String): String { + if (value.isBlank()) return "" + val channelId = Regex("[?&]channel_id=([^&#]+)").find(value)?.groupValues?.get(1) + return if (channelId != null) youtubeChannelUrl(channelId) else youtubeChannelUrl(value) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt new file mode 100644 index 00000000..e5bb46fb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.portability + +import java.io.OutputStream + +class PipePipePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.PIPE_PIPE, + 2, + newPipeArchiveCapabilities(), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val version = NewPipeArchiveDatabase.userVersion(input) ?: return null + if (version < PIPE_PIPE_DATABASE_VERSION) return null + return PortabilityDetection(PortabilityFormat.PIPE_PIPE, version.toString(), 100, "PipePipe SQLite schema version") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported PipePipe backup" } + NewPipeArchiveDatabase.read(input) { NewPipeDatabasePortabilityReader.read(it, sink) } + } + + override fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = super.assessExport(source, categories) + + newPipeArchiveExportIssues(source, categories, NewPipeArchiveTarget.PIPE_PIPE) + + override fun encode(source: PortabilityRecordSource, output: OutputStream, categories: Set) = + NewPipeArchivePortabilityWriter.write(source, output, categories, NewPipeArchiveTarget.PIPE_PIPE) + + private companion object { + const val PIPE_PIPE_DATABASE_VERSION = 900 + } +} + +internal fun archiveImportCapabilities(): Set = setOf( + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, +).mapTo(linkedSetOf()) { category -> + PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt new file mode 100644 index 00000000..02b3e783 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import java.io.OutputStream + +class PipedPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.PIPED, + 1, + setOf( + pipedCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.HISTORY, PortabilityFidelity.COMPLETE), + pipedCapability(PortabilityCategory.PLAYLISTS, PortabilityFidelity.PARTIAL), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + if (LIBRE_TUBE_FIELDS.any(probe::contains)) return null + if (probe.contains("\"format\"") && probe.contains("\"Piped\"")) { + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.PIPED, version, 96, "Piped format marker") + } + val trimmed = probe.trimStart() + if (trimmed.startsWith("[") && (probe.contains("UC") || probe.contains("youtube.com/channel/"))) { + return PortabilityDetection(PortabilityFormat.PIPED, null, 70, "Piped subscription array") + } + return null + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + when (parser.nextToken()) { + JsonToken.START_ARRAY -> PipedPortabilityReader.subscriptions(parser, sink) + JsonToken.START_OBJECT -> readObject(parser, sink) + else -> error("Invalid Piped backup") + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = PipedPortabilityWriter.write(source, output, categories) + + private fun readObject(parser: com.fasterxml.jackson.core.JsonParser, sink: PortabilityRecordSink) { + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions", "localSubscriptions" -> PipedPortabilityReader.subscriptions(parser, sink, token) + "groups", "channelGroups" -> PipedPortabilityReader.groups(parser, sink, token) + "watchHistory" -> PipedPortabilityReader.history(parser, sink, token) + "playlists" -> PipedPortabilityReader.playlists(parser, sink, token) + else -> parser.skipChildren() + } + } + require(format == "Piped" && version == 1) { "Unsupported Piped backup version" } + } + + private companion object { + val LIBRE_TUBE_FIELDS = listOf( + "\"watchPositions\"", + "\"customInstances\"", + "\"playlistBookmarks\"", + "\"localPlaylists\"", + ) + } +} + +private fun pipedCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt new file mode 100644 index 00000000..30703bf9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject + +internal object PipedPortabilityReader { + fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken = JsonToken.START_ARRAY) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped subscriptions" } + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + readArray(parser) { element -> + val item = element as? JsonObject + val channel = item?.string("channelId") ?: item?.string("url") ?: element.stringValue() + if (channel.isNotBlank()) { + sink.write( + PortabilitySubscription( + youtubeChannelUrl(channel), + item?.string("name") ?: item?.string("channelName").orEmpty(), + item?.string("avatar") ?: item?.string("channelThumbnail").orEmpty(), + item?.long("subscribedAt") ?: 0L, + ), + ) + } + } + } + + fun groups(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped groups" } + sink.markCategory(PortabilityCategory.SUBSCRIPTION_GROUPS) + readArray(parser) { element -> + val item = element.jsonObject + val name = item.string("groupName").ifBlank { item.string("name") } + if (name.isBlank()) return@readArray + sink.write(PortabilitySubscriptionGroup(name)) + item.array("channels").forEach { channel -> + val value = channel.stringValue() + if (value.isNotBlank()) sink.write(PortabilitySubscriptionGroupMembership(name, youtubeChannelUrl(value))) + } + } + } + + fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped history" } + sink.markCategory(PortabilityCategory.HISTORY) + readArray(parser) { element -> + val item = element.jsonObject + val videoId = item.string("videoId") + if (videoId.isBlank()) return@readArray + sink.write( + PortabilityHistory( + PortabilityVideo( + youtubeVideoUrl(videoId), + item.string("title"), + item.string("thumbnail"), + item.long("duration"), + item.string("uploaderName"), + item.string("uploaderUrl"), + ), + item.long("watchedAt"), + item.long("currentTime"), + ), + ) + } + } + + fun playlists(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { + require(token == JsonToken.START_ARRAY) { "Invalid Piped playlists" } + sink.markCategory(PortabilityCategory.PLAYLISTS) + var index = 0 + readArray(parser) { element -> + val item = element.jsonObject + val name = item.string("name") + val sourceId = "piped:${index++}:${name.lowercase()}" + sink.write(PortabilityPlaylist(sourceId, name)) + item.array("videos").forEachIndexed { position, video -> + sink.write(PortabilityPlaylistVideo(sourceId, position, PortabilityVideo(youtubeVideoUrl(video.stringValue())))) + } + } + } + + private fun readArray(parser: JsonParser, block: (JsonElement) -> Unit) { + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "Backup section contains too many records" } + block(parser.readJsonElement()) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt new file mode 100644 index 00000000..304e6cde --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt @@ -0,0 +1,83 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import java.io.OutputStream + +internal object PipedPortabilityWriter { + fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", "Piped") + json.writeNumberField("version", 1) + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + playlists(json, source, categories) + json.writeEndObject() + } + } + + private fun subscriptions(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + json.writeString(youtubeId((record as PortabilitySubscription).channelUrl)) + } + json.writeEndArray() + } + + private fun groups(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("groups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("groupName", record.name) + json.writeArrayFieldStart("channels") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + json.writeString(youtubeId((child as PortabilitySubscriptionGroupMembership).channelUrl)) + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("watchHistory") + source.forEach(PortabilityCategory.HISTORY) { record -> + val item = record as PortabilityHistory + json.writeStartObject() + json.writeStringField("videoId", youtubeId(item.video.url)) + json.writeStringField("title", item.video.title) + json.writeStringField("uploaderName", item.video.channelName) + json.writeStringField("uploaderUrl", item.video.channelUrl) + json.writeStringField("thumbnail", item.video.thumbnailUrl) + json.writeNumberField("duration", item.video.durationSeconds) + json.writeNumberField("watchedAt", item.watchedAt) + json.writeNumberField("currentTime", item.positionSeconds) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun playlists(json: JsonGenerator, source: PortabilityRecordSource, categories: Set) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeStringField("type", "playlist") + json.writeStringField("visibility", "private") + json.writeArrayFieldStart("videos") + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + json.writeString((child as PortabilityPlaylistVideo).video.url) + } + json.writeEndArray() + json.writeEndObject() + } + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt new file mode 100644 index 00000000..10dec38b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.nio.file.Path +import kotlinx.serialization.Serializable + +data class PortabilityInput( + val path: Path, + val filename: String, + val contentType: String?, + val size: Long, + val probe: ByteArray, + val archive: PortabilityArchiveInventory?, +) + +data class PortabilityDetection( + val format: PortabilityFormat, + val formatVersion: String?, + val confidence: Int, + val evidence: String, +) + +interface PortabilityRecordSink { + fun markCategory(category: PortabilityCategory) + fun write(record: PortabilityRecord): PortabilityWriteResult + fun issue(issue: PortabilityIssue) + fun putLookup(namespace: String, key: String, value: String) + fun lookup(namespace: String, key: String): String? +} + +interface PortabilityRecordSource { + fun categories(): Set + fun counts(): Map + fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) + fun forEachChild(category: PortabilityCategory, parentKey: String, block: (PortabilityRecord) -> Unit) { + forEach(category) { record -> + if (record.parentKey() == parentKey.trim().lowercase()) block(record) + } + } +} + +interface PortabilityAdapter { + val descriptor: PortabilityAdapterDescriptor + val autoDetect: Boolean get() = true + fun detect(input: PortabilityInput): PortabilityDetection? + fun decode(input: PortabilityInput, sink: PortabilityRecordSink) + fun assessExport( + source: PortabilityRecordSource, + categories: Set, + ): List = unsupportedExportCategories(categories) + fun encode(source: PortabilityRecordSource, output: OutputStream, categories: Set) + + private fun unsupportedExportCategories(categories: Set): List { + val supported = descriptor.capabilities + .filter { PortabilityDirection.EXPORT in it.directions } + .mapTo(hashSetOf()) { it.category } + return (categories - supported).map { category -> + PortabilityIssue(category, "unsupported_export_category", "${category.wireName} cannot be exported") + } + } +} + +data class PortabilityWriteResult( + val inserted: Boolean, +) + +@Serializable +data class PortabilityIssue( + val category: PortabilityCategory?, + val code: String, + val message: String, + val count: Long = 1L, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt new file mode 100644 index 00000000..85868142 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import java.nio.file.Path +import java.util.zip.ZipFile + +data class PortabilityArchiveEntry( + val name: String, + val compressedSize: Long, + val expandedSize: Long, +) + +data class PortabilityArchiveInventory( + val entries: List, + val expandedBytes: Long, +) { + val names: Set = entries.mapTo(linkedSetOf()) { it.name } +} + +object PortabilityArchiveInspector { + fun inspect(path: Path): PortabilityArchiveInventory? { + if (!hasZipSignature(path)) return null + ZipFile(path.toFile()).use { zip -> + val entries = ArrayList() + var expandedTotal = 0L + val iterator = zip.entries().asIterator() + while (iterator.hasNext()) { + val entry = iterator.next() + require(entries.size < PortabilityLimits.MAX_ARCHIVE_ENTRIES) { "Archive contains too many entries" } + validateName(entry.name) + if (entry.isDirectory) continue + val expanded = entry.size + val compressed = entry.compressedSize + require(expanded in 0..PortabilityLimits.MAX_ARCHIVE_ENTRY_BYTES) { "Archive entry is too large" } + expandedTotal = Math.addExact(expandedTotal, expanded) + require(expandedTotal <= PortabilityLimits.MAX_ARCHIVE_EXPANDED_BYTES) { "Archive expands beyond the limit" } + if (expanded > 0L && compressed == 0L) error("Archive entry has an invalid compression size") + if (expanded > 0L && compressed > 0L) { + require(expanded / compressed <= PortabilityLimits.MAX_COMPRESSION_RATIO) { + "Archive entry exceeds the compression ratio limit" + } + } + entries += PortabilityArchiveEntry(entry.name, compressed, expanded) + } + return PortabilityArchiveInventory(entries, expandedTotal) + } + } + + private fun hasZipSignature(path: Path): Boolean { + if (path.toFile().length() < 4L) return false + return path.toFile().inputStream().use { input -> + input.read() == 0x50 && input.read() == 0x4b + } + } + + private fun validateName(name: String) { + require(name.isNotBlank() && '\u0000' !in name) { "Archive entry has an invalid name" } + val normalized = Path.of(name.replace('\\', '/')).normalize() + require(!normalized.isAbsolute && normalized.none { it.toString() == ".." }) { + "Archive entry escapes its root" + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt new file mode 100644 index 00000000..0757eb72 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.portability + +interface PortabilityDataPort { + suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + ): Map + + suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt new file mode 100644 index 00000000..9d4d83a2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -0,0 +1,209 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path + +class PortabilityEngine internal constructor( + private val registry: PortabilityRegistry, + private val dataPort: PortabilityDataPort, + private val store: PortabilityJobStore, + private val scope: CoroutineScope, +) : AutoCloseable { + fun formats(): List = registry.descriptors() + + fun startImportPreview( + userId: String, + upload: Path, + filename: String, + contentType: String?, + formatHint: PortabilityFormat? = null, + requestId: String? = null, + ): PortabilityJobSnapshot { + val job = store.create(userId, PortabilityJobKind.IMPORT, requestId) + val saved = job.directory.resolve("upload") + try { + Files.move(upload, saved) + } catch (error: Exception) { + store.remove(userId, job.id) + throw error + } + job.task = scope.launch { analyze(job, saved, filename, contentType, formatHint) } + return job.snapshot() + } + + fun startExport( + userId: String, + format: PortabilityFormat, + categories: Set, + requestId: String? = null, + ): PortabilityJobSnapshot { + require(categories.isNotEmpty()) { "At least one category is required" } + val job = store.create(userId, PortabilityJobKind.EXPORT, requestId) + job.task = scope.launch { export(job, format, categories) } + return job.snapshot() + } + + fun snapshot(userId: String, id: String): PortabilityJobSnapshot = store.get(userId, id).snapshot() + + fun report(userId: String, id: String): PortabilityJobReport = store.get(userId, id).report() + + fun artifact(userId: String, id: String): Path { + val job = store.get(userId, id) + check(job.snapshot().state == PortabilityJobState.COMPLETED) { "Export is not complete" } + return requireNotNull(job.artifact) { "Export artifact is unavailable" } + } + + fun applyImport(userId: String, id: String, request: PortabilityImportRequest): PortabilityJobSnapshot { + require(request.categories.isNotEmpty()) { "At least one category is required" } + val job = store.get(userId, id) + val preview = requireNotNull(job.snapshot().preview) { "Import preview is not ready" } + require(request.categories.all { it.wireName in preview.counts }) { "A selected category is unavailable" } + job.transition(setOf(PortabilityJobState.READY), PortabilityJobState.APPLYING) + job.task = scope.launch { apply(job, request) } + return job.snapshot() + } + + fun cancel(userId: String, id: String): PortabilityJobSnapshot { + val job = store.get(userId, id) + val state = job.snapshot().state + if (state in TERMINAL_STATES) return job.snapshot() + if (job.tryTransition(ACTIVE_STATES, PortabilityJobState.CANCELLED)) job.task?.cancel() + return job.snapshot() + } + + fun delete(userId: String, id: String) { + val job = store.get(userId, id) + check(!job.isCancelled() || job.isTaskComplete()) { "Cancelled job is still stopping" } + store.remove(userId, id) + } + + override fun close() { + scope.cancel() + store.close() + } + + private suspend fun analyze( + job: PortabilityJob, + upload: Path, + filename: String, + contentType: String?, + formatHint: PortabilityFormat?, + ) { + try { + runJob(job, PortabilityJobState.ANALYZING) { + val input = withContext(Dispatchers.IO) { PortabilityInputFactory.create(upload, filename, contentType) } + val (adapter, detection) = registry.detect(input, formatHint) + val spool = PortabilitySpool.create(job.directory) + job.spool = spool + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.ANALYZING, + PortabilityProgressUnit.RECORDS, + ) + withContext(Dispatchers.IO) { adapter.decode(input, ProgressRecordSink(spool, progress)) } + progress.finish() + val preview = PortabilityPreview( + detection = detection.toItem(adapter.descriptor.adapterVersion), + counts = spool.counts().mapKeys { it.key.wireName }, + duplicates = spool.duplicateCount(), + issues = spool.issues(), + ) + job.transition(setOf(PortabilityJobState.ANALYZING), PortabilityJobState.READY, preview) + } + } finally { + Files.deleteIfExists(upload) + } + } + + private suspend fun apply(job: PortabilityJob, request: PortabilityImportRequest) { + runJob(job, null) { + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.CATEGORIES, + request.categories.size.toLong(), + interval = 1L, + ) + val result = dataPort.import(job.ownerId, requireNotNull(job.spool), request) { _, _ -> progress.add() } + progress.finish() + job.transition(setOf(PortabilityJobState.APPLYING), PortabilityJobState.COMPLETED, result = result) + } + } + + private suspend fun export(job: PortabilityJob, format: PortabilityFormat, categories: Set) { + runJob(job, PortabilityJobState.ENCODING) { + val adapter = registry.adapter(format) + val spool = PortabilitySpool.create(job.directory) + job.spool = spool + val collecting = PortabilityProgressReporter( + job, + PortabilityProgressPhase.COLLECTING, + PortabilityProgressUnit.RECORDS, + ) + dataPort.export(job.ownerId, categories, ProgressRecordSink(spool, collecting)) + collecting.finish() + val issues = adapter.assessExport(spool, categories) + require(issues.none { it.code == "unsupported_export_category" }) { "Export category is unsupported" } + val artifact = job.directory.resolve("export.${adapter.descriptor.defaultExtension}") + val encoding = PortabilityProgressReporter( + job, + PortabilityProgressPhase.ENCODING, + PortabilityProgressUnit.BYTES, + interval = 64L * 1024L, + ) + withContext(Dispatchers.IO) { + Files.newOutputStream(artifact).use { raw -> + ProgressOutputStream(raw, encoding).buffered().use { adapter.encode(spool, it, categories) } + } + } + encoding.finish() + job.artifact = artifact + job.transition( + setOf(PortabilityJobState.ENCODING), + PortabilityJobState.COMPLETED, + preview = PortabilityPreview( + PortabilityDetectionItem(format, null, adapter.descriptor.adapterVersion, 100, "Requested export format"), + spool.counts().mapKeys { it.key.wireName }, + spool.duplicateCount(), + issues, + ), + ) + } + } + + private suspend fun runJob(job: PortabilityJob, initial: PortabilityJobState?, block: suspend () -> Unit) { + try { + initial?.let { job.transition(setOf(PortabilityJobState.QUEUED), it) } + block() + } catch (error: CancellationException) { + job.tryTransition(ACTIVE_STATES, PortabilityJobState.CANCELLED) + throw error + } catch (error: Exception) { + job.fail(error) + } + } + + private companion object { + val TERMINAL_STATES = setOf( + PortabilityJobState.COMPLETED, + PortabilityJobState.FAILED, + PortabilityJobState.CANCELLED, + ) + val ACTIVE_STATES = PortabilityJobState.entries.toSet() - TERMINAL_STATES + } +} + +private fun PortabilityDetection.toItem(adapterVersion: Int) = PortabilityDetectionItem( + format, + formatVersion, + adapterVersion, + confidence, + evidence, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt new file mode 100644 index 00000000..107da5c7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CoroutineScope +import java.nio.file.Path + +object PortabilityEngineFactory { + fun create(root: Path, scope: CoroutineScope): PortabilityEngine { + val adapters = listOf( + TypeTypePortabilityAdapter(), + PipePipePortabilityAdapter(), + NewPipePortabilityAdapter(), + InvidiousPortabilityAdapter(), + PipedPortabilityAdapter(), + LibreTubePortabilityAdapter(), + ViewTubePortabilityAdapter(), + FlowPortabilityAdapter(), + GrayjayPortabilityAdapter(), + YoutubeTakeoutPortabilityAdapter(), + OpmlPortabilityAdapter(), + MaterialiousPortabilityAdapter(), + OpmlPortabilityAdapter(PortabilityFormat.SKY_TUBE, autoDetect = false), + OpmlPortabilityAdapter(PortabilityFormat.YOUTUBE_LOCAL, autoDetect = false), + ) + require(adapters.map { it.descriptor.format }.toSet() == PortabilityFormat.entries.toSet()) { + "Every portability format must have an adapter" + } + return PortabilityEngine( + registry = PortabilityRegistry(adapters), + dataPort = TypeTypePortabilityDataPort(), + store = PortabilityJobStore(root), + scope = scope, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt new file mode 100644 index 00000000..b94685bb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt @@ -0,0 +1,94 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +enum class PortabilityFormat(val wireName: String) { + @SerialName("typetype") + TYPE_TYPE("typetype"), + @SerialName("pipepipe") + PIPE_PIPE("pipepipe"), + @SerialName("newpipe") + NEW_PIPE("newpipe"), + @SerialName("invidious") + INVIDIOUS("invidious"), + @SerialName("piped") + PIPED("piped"), + @SerialName("libretube") + LIBRE_TUBE("libretube"), + @SerialName("viewtube") + VIEW_TUBE("viewtube"), + @SerialName("materialious") + MATERIALIOUS("materialious"), + @SerialName("youtube-local") + YOUTUBE_LOCAL("youtube-local"), + @SerialName("flow") + FLOW("flow"), + @SerialName("skytube") + SKY_TUBE("skytube"), + @SerialName("grayjay") + GRAYJAY("grayjay"), + @SerialName("youtube-takeout") + YOUTUBE_TAKEOUT("youtube-takeout"), + @SerialName("opml") + OPML("opml"), +} + +@Serializable +enum class PortabilityCategory(val wireName: String) { + @SerialName("subscriptions") + SUBSCRIPTIONS("subscriptions"), + @SerialName("subscriptionGroups") + SUBSCRIPTION_GROUPS("subscriptionGroups"), + @SerialName("history") + HISTORY("history"), + @SerialName("playlists") + PLAYLISTS("playlists"), + @SerialName("watchLater") + WATCH_LATER("watchLater"), + @SerialName("favorites") + FAVORITES("favorites"), + @SerialName("progress") + PROGRESS("progress"), + @SerialName("searchHistory") + SEARCH_HISTORY("searchHistory"), + @SerialName("savedPlaylists") + SAVED_PLAYLISTS("savedPlaylists"), + @SerialName("settings") + SETTINGS("settings"), + @SerialName("contentFilters") + CONTENT_FILTERS("contentFilters"), +} + +@Serializable +enum class PortabilityDirection { + @SerialName("import") + IMPORT, + @SerialName("export") + EXPORT, +} + +@Serializable +data class PortabilityCapability( + val category: PortabilityCategory, + val directions: Set, + val fidelity: PortabilityFidelity, +) + +@Serializable +enum class PortabilityFidelity { + @SerialName("complete") + COMPLETE, + @SerialName("partial") + PARTIAL, +} + +@Serializable +data class PortabilityAdapterDescriptor( + val format: PortabilityFormat, + val adapterVersion: Int, + val capabilities: Set, + val defaultExtension: String, + val contentType: String, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt new file mode 100644 index 00000000..4ef79671 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path + +object PortabilityInputFactory { + fun create(path: Path, filename: String, contentType: String?): PortabilityInput { + val size = Files.size(path) + require(size in 1..PortabilityLimits.MAX_UPLOAD_BYTES) { "Backup file size is outside the allowed range" } + val probe = Files.newInputStream(path).use { it.readNBytes(PortabilityLimits.PROBE_BYTES) } + return PortabilityInput( + path = path, + filename = filename, + contentType = contentType, + size = size, + probe = probe, + archive = PortabilityArchiveInspector.inspect(path), + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt new file mode 100644 index 00000000..100ecf63 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -0,0 +1,120 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.Job +import org.slf4j.LoggerFactory +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicReference + +internal class PortabilityJob( + val id: String, + val ownerId: String, + val kind: PortabilityJobKind, + val directory: Path, + val requestId: String?, + private val clock: () -> Long, +) { + val createdAt = clock() + private val value = AtomicReference( + PortabilityJobSnapshot(id, kind, PortabilityJobState.QUEUED, createdAt, createdAt, requestId), + ) + @Volatile + var task: Job? = null + @Volatile + var spool: PortabilitySpool? = null + @Volatile + var artifact: Path? = null + + fun snapshot(): PortabilityJobSnapshot = value.get() + + fun report(): PortabilityJobReport = value.get().let { + PortabilityJobReport(it.id, it.state, it.requestId, it.preview, it.result, it.errorCode, it.errorMessage) + } + + fun isTerminal(): Boolean = value.get().state in TERMINAL_STATES + + fun isCancelled(): Boolean = value.get().state == PortabilityJobState.CANCELLED + + fun isTaskComplete(): Boolean = task?.isCompleted != false + + fun updateProgress(progress: PortabilityJobProgress) { + while (true) { + val current = value.get() + if (current.state in TERMINAL_STATES) return + val next = current.copy(updatedAt = clock(), progress = progress) + if (value.compareAndSet(current, next)) return + } + } + + fun transition( + expected: Set, + state: PortabilityJobState, + preview: PortabilityPreview? = value.get().preview, + result: Map? = value.get().result, + errorCode: String? = null, + errorMessage: String? = null, + ) { + while (true) { + val current = value.get() + check(current.state in expected) { "Invalid portability transition ${current.state} -> $state" } + val next = current.copy( + state = state, + updatedAt = clock(), + preview = preview, + result = result, + progress = current.progress, + errorCode = errorCode, + errorMessage = errorMessage, + ) + if (value.compareAndSet(current, next)) return + } + } + + fun tryTransition( + expected: Set, + state: PortabilityJobState, + errorCode: String? = null, + errorMessage: String? = null, + ): Boolean { + while (true) { + val current = value.get() + if (current.state !in expected) return false + val next = current.copy( + state = state, + updatedAt = clock(), + errorCode = errorCode, + errorMessage = errorMessage, + ) + if (value.compareAndSet(current, next)) return true + } + } + + fun fail(error: Exception) { + val code = portabilityErrorCode(error) + logger.error("Portability job failed jobId={} requestId={} code={}", id, requestId ?: "none", code, error) + tryTransition( + PortabilityJobState.entries.toSet() - TERMINAL_STATES, + PortabilityJobState.FAILED, + errorCode = code, + errorMessage = portabilityErrorMessage(error), + ) + } + + fun delete() { + task?.cancel() + spool?.delete() + if (!Files.exists(directory)) return + Files.walk(directory).use { paths -> + paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + + private companion object { + val logger = LoggerFactory.getLogger(PortabilityJob::class.java) + val TERMINAL_STATES = setOf( + PortabilityJobState.COMPLETED, + PortabilityJobState.FAILED, + PortabilityJobState.CANCELLED, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt new file mode 100644 index 00000000..97fffcfa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt @@ -0,0 +1,125 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +enum class PortabilityJobKind { + @SerialName("import") + IMPORT, + @SerialName("export") + EXPORT, +} + +@Serializable +enum class PortabilityJobState { + @SerialName("queued") + QUEUED, + @SerialName("analyzing") + ANALYZING, + @SerialName("ready") + READY, + @SerialName("applying") + APPLYING, + @SerialName("encoding") + ENCODING, + @SerialName("completed") + COMPLETED, + @SerialName("failed") + FAILED, + @SerialName("cancelled") + CANCELLED, +} + +@Serializable +enum class PortabilityProgressPhase { + @SerialName("analyzing") + ANALYZING, + @SerialName("collecting") + COLLECTING, + @SerialName("applying") + APPLYING, + @SerialName("encoding") + ENCODING, +} + +@Serializable +enum class PortabilityProgressUnit { + @SerialName("records") + RECORDS, + @SerialName("categories") + CATEGORIES, + @SerialName("bytes") + BYTES, +} + +@Serializable +data class PortabilityJobProgress( + val phase: PortabilityProgressPhase, + val unit: PortabilityProgressUnit, + val processed: Long, + val total: Long? = null, +) + +@Serializable +data class PortabilityPreview( + val detection: PortabilityDetectionItem, + val counts: Map, + val duplicates: Long, + val issues: List, +) + +@Serializable +data class PortabilityDetectionItem( + val format: PortabilityFormat, + val formatVersion: String?, + val adapterVersion: Int, + val confidence: Int, + val evidence: String, +) + +@Serializable +data class PortabilityJobSnapshot( + val id: String, + val kind: PortabilityJobKind, + val state: PortabilityJobState, + val createdAt: Long, + val updatedAt: Long, + val requestId: String? = null, + val preview: PortabilityPreview? = null, + val result: Map? = null, + val progress: PortabilityJobProgress? = null, + val errorCode: String? = null, + val errorMessage: String? = null, +) + +@Serializable +data class PortabilityJobReport( + val id: String, + val state: PortabilityJobState, + val requestId: String? = null, + val preview: PortabilityPreview? = null, + val result: Map? = null, + val errorCode: String? = null, + val errorMessage: String? = null, +) + +@Serializable +data class PortabilityImportRequest( + val categories: Set, + val duplicatePolicy: PortabilityDuplicatePolicy = PortabilityDuplicatePolicy.SKIP, +) + +@Serializable +data class PortabilityExportRequest( + val format: PortabilityFormat, + val categories: Set, +) + +@Serializable +enum class PortabilityDuplicatePolicy { + @SerialName("skip") + SKIP, + @SerialName("replace") + REPLACE, +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt new file mode 100644 index 00000000..feb3eae2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap + +internal class PortabilityJobStore( + private val root: Path, + private val clock: () -> Long = System::currentTimeMillis, + private val retentionMs: Long = DEFAULT_RETENTION_MS, +) : AutoCloseable { + private val jobs = ConcurrentHashMap() + + init { + Files.createDirectories(root) + } + + fun create(ownerId: String, kind: PortabilityJobKind, requestId: String? = null): PortabilityJob { + cleanup() + val id = UUID.randomUUID().toString() + val directory = Files.createDirectory(root.resolve(id)) + return PortabilityJob(id, ownerId, kind, directory, requestId, clock).also { jobs[id] = it } + } + + fun get(ownerId: String, id: String): PortabilityJob = jobs[id] + ?.takeIf { it.ownerId == ownerId } + ?: throw PortabilityJobNotFoundException() + + fun remove(ownerId: String, id: String) { + val job = get(ownerId, id) + if (jobs.remove(id, job)) job.delete() + } + + fun cleanup() { + val oldest = clock() - retentionMs + jobs.entries.removeIf { entry -> + val expired = entry.value.isTerminal() && entry.value.isTaskComplete() && entry.value.snapshot().updatedAt < oldest + if (expired) entry.value.delete() + expired + } + } + + override fun close() { + jobs.values.forEach(PortabilityJob::delete) + jobs.clear() + Files.deleteIfExists(root) + } + + private companion object { + const val DEFAULT_RETENTION_MS = 24L * 60L * 60L * 1_000L + } +} + +class PortabilityJobNotFoundException : NoSuchElementException("Portability job not found") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt new file mode 100644 index 00000000..8e0f00b1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt @@ -0,0 +1,68 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import com.fasterxml.jackson.core.StreamReadConstraints +import com.fasterxml.jackson.core.StreamReadFeature +import dev.typetype.server.cache.CacheJson +import kotlinx.serialization.json.JsonElement +import java.nio.file.Files +import java.io.ByteArrayOutputStream + +internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() + .streamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(100) + .maxStringLength(PortabilityLimits.MAX_RECORD_JSON_BYTES) + .maxNumberLength(1_000) + .build(), + ) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build() + +internal inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = + Files.newInputStream(path).buffered().use { input -> + PortabilityJsonFactory.createParser(input).use(block) + } + +internal fun JsonParser.requireObject() { + require(nextToken() == JsonToken.START_OBJECT) { "Backup root must be a JSON object" } +} + +internal fun JsonParser.textOrEmpty(): String = + if (currentToken().isScalarValue) valueAsString.orEmpty() else "" + +internal fun JsonParser.longOrZero(): Long = + if (currentToken().isNumeric) longValue else valueAsString?.toLongOrNull() ?: 0L + +internal fun JsonParser.readJsonElement(): JsonElement { + val bytes = ByteArrayOutputStream() + PortabilityJsonFactory.createGenerator(bytes).use { generator -> + generator.copyCurrentStructure(this) + } + require(bytes.size() <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "JSON value is too large" } + return CacheJson.parseToJsonElement(bytes.toString(Charsets.UTF_8)) +} + +internal fun youtubeChannelUrl(value: String): String { + val trimmed = value.trim() + return when { + trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed + trimmed.isNotBlank() -> "https://www.youtube.com/channel/$trimmed" + else -> "" + } +} + +internal fun youtubeVideoUrl(value: String): String { + val trimmed = value.trim() + return when { + trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed + trimmed.isNotBlank() -> "https://www.youtube.com/watch?v=$trimmed" + else -> "" + } +} + +internal fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { + it != '&' && it != '?' && it != '#' +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt new file mode 100644 index 00000000..a6d0dc0d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt @@ -0,0 +1,31 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive + +internal fun JsonObject.string(name: String): String = get(name)?.let { element -> + runCatching { element.jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") +}.orEmpty() + +internal fun JsonObject.long(name: String): Long = get(name)?.let { element -> + runCatching { element.jsonPrimitive.contentOrNull?.toLongOrNull() ?: 0L }.getOrDefault(0L) +} ?: 0L + +internal fun JsonObject.int(name: String): Int = long(name).toInt() + +internal fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> + runCatching { element.jsonArray }.getOrNull() +} ?: JsonArray(emptyList()) + +internal fun JsonElement.stringValue(): String = runCatching { + jsonPrimitive.contentOrNull.orEmpty() +}.getOrDefault("") + +internal fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject + +internal fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt new file mode 100644 index 00000000..0d5a5aa3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt @@ -0,0 +1,13 @@ +package dev.typetype.server.portability + +object PortabilityLimits { + const val MAX_UPLOAD_BYTES = 512L * 1024 * 1024 + const val MAX_ARCHIVE_ENTRIES = 10_000 + const val MAX_ARCHIVE_EXPANDED_BYTES = 2L * 1024 * 1024 * 1024 + const val MAX_ARCHIVE_ENTRY_BYTES = 512L * 1024 * 1024 + const val MAX_COMPRESSION_RATIO = 200L + const val PROBE_BYTES = 64 * 1024 + const val MAX_RECORDS = 2_000_000L + const val MAX_RECORD_JSON_BYTES = 2 * 1024 * 1024 + const val MAX_CONTAINER_RECORDS = 100_000 +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt new file mode 100644 index 00000000..265850d2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -0,0 +1,92 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CancellationException +import java.io.FilterOutputStream +import java.io.OutputStream + +internal class PortabilityProgressReporter( + private val job: PortabilityJob, + private val phase: PortabilityProgressPhase, + private val unit: PortabilityProgressUnit, + private val total: Long? = null, + private val interval: Long = 100L, +) { + private var processed = 0L + private var published = -1L + + init { + publish(force = true) + } + + fun add(count: Long = 1L) { + ensureActive() + require(count >= 0L) + processed = Math.addExact(processed, count) + publish(force = false) + } + + fun finish() { + ensureActive() + publish(force = true) + } + + fun ensureActive() { + if (job.isCancelled()) throw CancellationException("Portability job was cancelled") + } + + private fun publish(force: Boolean) { + if (!force && processed - published < interval) return + job.updateProgress(PortabilityJobProgress(phase, unit, processed, total)) + published = processed + } +} + +internal class ProgressRecordSink( + private val delegate: PortabilityRecordSink, + private val progress: PortabilityProgressReporter, +) : PortabilityRecordSink { + override fun markCategory(category: PortabilityCategory) { + progress.ensureActive() + delegate.markCategory(category) + } + + override fun write(record: PortabilityRecord): PortabilityWriteResult { + progress.ensureActive() + return delegate.write(record).also { progress.add() } + } + + override fun issue(issue: PortabilityIssue) { + progress.ensureActive() + delegate.issue(issue) + } + + override fun putLookup(namespace: String, key: String, value: String) { + progress.ensureActive() + delegate.putLookup(namespace, key, value) + } + + override fun lookup(namespace: String, key: String): String? { + progress.ensureActive() + return delegate.lookup(namespace, key) + } + + fun count(category: PortabilityCategory): Long = + (delegate as? PortabilityRecordSource)?.counts()?.get(category) ?: 0L +} + +internal class ProgressOutputStream( + output: OutputStream, + private val progress: PortabilityProgressReporter, +) : FilterOutputStream(output) { + override fun write(value: Int) { + progress.ensureActive() + out.write(value) + progress.add() + } + + override fun write(buffer: ByteArray, offset: Int, length: Int) { + progress.ensureActive() + out.write(buffer, offset, length) + progress.add(length.toLong()) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt new file mode 100644 index 00000000..0f7b6238 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt @@ -0,0 +1,160 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject + +@Serializable +sealed interface PortabilityRecord { + val category: PortabilityCategory + fun stableKey(): String + fun parentKey(): String? = null +} + +@Serializable +@SerialName("subscription") +data class PortabilitySubscription( + val channelUrl: String, + val name: String = "", + val avatarUrl: String = "", + val subscribedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTIONS + override fun stableKey() = channelUrl.trim().lowercase() +} + +@Serializable +@SerialName("subscriptionGroup") +data class PortabilitySubscriptionGroup( + val name: String, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTION_GROUPS + override fun stableKey() = "group:${name.trim().lowercase()}" +} + +@Serializable +@SerialName("subscriptionGroupMembership") +data class PortabilitySubscriptionGroupMembership( + val groupName: String, + val channelUrl: String, +) : PortabilityRecord { + override val category = PortabilityCategory.SUBSCRIPTION_GROUPS + override fun stableKey() = "member:${groupName.trim().lowercase()}:${channelUrl.trim().lowercase()}" + override fun parentKey() = groupName.trim().lowercase() +} + +@Serializable +@SerialName("history") +data class PortabilityHistory( + val video: PortabilityVideo, + val watchedAt: Long, + val positionSeconds: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.HISTORY + override fun stableKey() = "${video.url.trim().lowercase()}:$watchedAt" +} + +@Serializable +@SerialName("playlist") +data class PortabilityPlaylist( + val sourceId: String, + val name: String, + val description: String = "", + val createdAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PLAYLISTS + override fun stableKey() = "playlist:${sourceId.ifBlank { name }.trim().lowercase()}" +} + +@Serializable +@SerialName("playlistVideo") +data class PortabilityPlaylistVideo( + val playlistSourceId: String, + val position: Int, + val video: PortabilityVideo, + val addedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PLAYLISTS + override fun stableKey() = "playlist-video:${playlistSourceId.trim().lowercase()}:$position" + override fun parentKey() = playlistSourceId.trim().lowercase() +} + +@Serializable +@SerialName("watchLater") +data class PortabilityWatchLater( + val video: PortabilityVideo, + val addedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.WATCH_LATER + override fun stableKey() = video.url.trim().lowercase() +} + +@Serializable +@SerialName("favorite") +data class PortabilityFavorite( + val video: PortabilityVideo, + val favoritedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.FAVORITES + override fun stableKey() = video.url.trim().lowercase() +} + +@Serializable +@SerialName("progress") +data class PortabilityProgress( + val videoUrl: String, + val positionSeconds: Long, + val updatedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.PROGRESS + override fun stableKey() = videoUrl.trim().lowercase() +} + +@Serializable +@SerialName("searchHistory") +data class PortabilitySearchHistory( + val term: String, + val searchedAt: Long, +) : PortabilityRecord { + override val category = PortabilityCategory.SEARCH_HISTORY + override fun stableKey() = "${term.trim().lowercase()}:$searchedAt" +} + +@Serializable +@SerialName("savedPlaylist") +data class PortabilitySavedPlaylist( + val sourceId: String, + val url: String, + val title: String = "", + val thumbnailUrl: String = "", + val uploaderName: String = "", + val streamCount: Long = 0L, + val playlistType: String = "", + val savedAt: Long = 0L, +) : PortabilityRecord { + override val category = PortabilityCategory.SAVED_PLAYLISTS + override fun stableKey() = url.trim().lowercase() +} + +@Serializable +@SerialName("settings") +data class PortabilitySettings( + val values: JsonObject, +) : PortabilityRecord { + override val category = PortabilityCategory.SETTINGS + override fun stableKey() = "settings" +} + +@Serializable +@SerialName("contentFilter") +data class PortabilityContentFilter( + val kind: String, + val value: String, + val label: String = "", + val imageUrl: String = "", + val createdAt: Long = 0L, + val metadata: JsonObject = JsonObject(emptyMap()), +) : PortabilityRecord { + override val category = PortabilityCategory.CONTENT_FILTERS + override fun stableKey() = "${kind.trim().lowercase()}:${value.trim().lowercase()}" +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt new file mode 100644 index 00000000..7ffa01a1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt @@ -0,0 +1,78 @@ +package dev.typetype.server.portability + +class PortabilityRegistry(adapters: List) { + private val adapters = adapters.toList() + private val byFormat = adapters.associateBy { it.descriptor.format } + + init { + require(adapters.isNotEmpty()) { "At least one portability adapter is required" } + require(byFormat.size == adapters.size) { "Only one adapter per format can be registered" } + } + + fun descriptors(): List = adapters.map { it.descriptor } + + fun adapter(format: PortabilityFormat): PortabilityAdapter = + byFormat[format] ?: throw UnsupportedPortabilityFormatException(format) + + fun detect( + input: PortabilityInput, + formatHint: PortabilityFormat? = null, + ): Pair { + if (formatHint != null) { + val adapter = adapter(formatHint) + val detection = adapter.detect(input) + ?: throw PortabilityFormatMismatchException(formatHint) + return adapter to detection + } + val matches = adapters.asSequence() + .filter(PortabilityAdapter::autoDetect) + .mapNotNull { adapter -> adapter.detect(input)?.let { adapter to it } } + .sortedByDescending { it.second.confidence } + .toList() + val best = matches.firstOrNull() ?: throw UnknownPortabilityFormatException() + require(best.second.confidence in 1..100) { "Adapter returned an invalid confidence" } + val second = matches.getOrNull(1) + if (second != null && best.second.confidence - second.second.confidence < MIN_CONFIDENCE_GAP) { + throw AmbiguousPortabilityFormatException(best.second, second.second) + } + return best + } + + private companion object { + const val MIN_CONFIDENCE_GAP = 10 + } +} + +sealed class PortabilityContractException(val code: String, message: String) : IllegalArgumentException(message) + +class UnknownPortabilityFormatException : + PortabilityContractException("portability_format_unknown", "Unable to detect the backup format") + +class UnsupportedPortabilityFormatException(format: PortabilityFormat) : + PortabilityContractException("portability_format_unsupported", "No adapter is registered for ${format.wireName}") + +class AmbiguousPortabilityFormatException(first: PortabilityDetection, second: PortabilityDetection) : + PortabilityContractException( + "portability_format_ambiguous", + "Backup matches both ${first.format.wireName} and ${second.format.wireName}", + ) + +class PortabilityFormatMismatchException(format: PortabilityFormat) : + PortabilityContractException("portability_format_mismatch", "Backup does not match ${format.wireName}") + +class PortabilityUploadTooLargeException : + PortabilityContractException("portability_upload_too_large", "Backup exceeds the upload limit") + +internal fun portabilityErrorCode(error: Exception): String = when (error) { + is PortabilityContractException -> error.code + is PortabilityJobNotFoundException -> "portability_job_not_found" + is IllegalStateException -> "portability_invalid_state" + is IllegalArgumentException -> "portability_invalid_input" + else -> "portability_failed" +} + +internal fun portabilityErrorMessage(error: Exception): String = when (error) { + is PortabilityContractException, is IllegalArgumentException -> + error.message ?: "Invalid portability data" + else -> "Portability operation failed" +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt new file mode 100644 index 00000000..24acc1d5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt @@ -0,0 +1,216 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import kotlinx.serialization.encodeToString +import java.io.Closeable +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.util.EnumSet + +class PortabilitySpool private constructor( + val path: Path, + private val connection: Connection, +) : PortabilityRecordSink, PortabilityRecordSource, Closeable { + private val insertRecord = connection.prepareStatement( + "INSERT OR IGNORE INTO records(category, stable_key, parent_key, payload) VALUES (?, ?, ?, ?)", + ) + private val markCategory = connection.prepareStatement( + "INSERT OR IGNORE INTO categories(category) VALUES (?)", + ) + private val upsertIssue = connection.prepareStatement( + """ + INSERT INTO issues(category, code, message, item_count) VALUES (?, ?, ?, ?) + ON CONFLICT(category, code, message) DO UPDATE SET item_count = item_count + excluded.item_count + """.trimIndent(), + ) + private val putLookup = connection.prepareStatement( + "INSERT OR REPLACE INTO lookups(namespace, lookup_key, value) VALUES (?, ?, ?)", + ) + private val getLookup = connection.prepareStatement( + "SELECT value FROM lookups WHERE namespace = ? AND lookup_key = ?", + ) + private val markedCategories = EnumSet.noneOf(PortabilityCategory::class.java) + private var pendingWrites = 0 + private var attemptedRecords = 0L + private var closed = false + + override fun markCategory(category: PortabilityCategory) { + checkOpen() + if (!markedCategories.add(category)) return + markCategory.setString(1, category.wireName) + markCategory.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun write(record: PortabilityRecord): PortabilityWriteResult { + checkOpen() + markCategory(record.category) + require(attemptedRecords < PortabilityLimits.MAX_RECORDS) { "Backup contains too many records" } + val payload = CacheJson.encodeToString(record) + require(payload.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { + "Backup record is too large" + } + insertRecord.setString(1, record.category.wireName) + insertRecord.setString(2, portabilityStableHash(record.stableKey())) + insertRecord.setString(3, record.parentKey()?.let(::portabilityStableHash)) + insertRecord.setString(4, payload) + attemptedRecords += 1 + val inserted = insertRecord.executeUpdate() == 1 + pendingWrites += 1 + flushWhenNeeded() + return PortabilityWriteResult(inserted) + } + + override fun issue(issue: PortabilityIssue) { + checkOpen() + require(issue.code.isNotBlank() && issue.message.isNotBlank() && issue.count > 0L) + upsertIssue.setString(1, issue.category?.wireName ?: "") + upsertIssue.setString(2, issue.code) + upsertIssue.setString(3, issue.message) + upsertIssue.setLong(4, issue.count) + upsertIssue.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun putLookup(namespace: String, key: String, value: String) { + checkOpen() + require(namespace.isNotBlank() && key.isNotBlank()) + require(value.toByteArray().size <= PortabilityLimits.MAX_RECORD_JSON_BYTES) { "Lookup value is too large" } + putLookup.setString(1, namespace) + putLookup.setString(2, portabilityStableHash(key.trim().lowercase())) + putLookup.setString(3, value) + putLookup.executeUpdate() + pendingWrites += 1 + flushWhenNeeded() + } + + override fun lookup(namespace: String, key: String): String? { + checkOpen() + flush() + getLookup.setString(1, namespace) + getLookup.setString(2, portabilityStableHash(key.trim().lowercase())) + return getLookup.executeQuery().use { rows -> if (rows.next()) rows.getString(1) else null } + } + + override fun categories(): Set = counts().keys + + override fun counts(): Map { + flush() + connection.createStatement().use { statement -> + statement.executeQuery( + """ + SELECT categories.category, COUNT(records.ordinal) + FROM categories + LEFT JOIN records ON records.category = categories.category + GROUP BY categories.category + ORDER BY categories.category + """.trimIndent(), + ).use { rows -> + return buildMap { + while (rows.next()) { + val category = portabilityCategoryByWireName(rows.getString(1)) + put(category, rows.getLong(2)) + } + } + } + } + } + + fun duplicateCount(): Long = attemptedRecords - counts().values.sum() + + fun issues(): List { + flush() + connection.createStatement().use { statement -> + statement.executeQuery( + "SELECT category, code, message, item_count FROM issues ORDER BY rowid", + ).use { rows -> + return buildList { + while (rows.next()) { + add( + PortabilityIssue( + category = rows.getString(1) + .takeIf(String::isNotBlank) + ?.let(::portabilityCategoryByWireName), + code = rows.getString(2), + message = rows.getString(3), + count = rows.getLong(4), + ), + ) + } + } + } + } + } + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + flush() + connection.prepareStatement( + "SELECT payload FROM records WHERE category = ? ORDER BY ordinal", + ).use { statement -> + statement.setString(1, category.wireName) + statement.executeQuery().use { rows -> + while (rows.next()) block(CacheJson.decodeFromString(rows.getString(1))) + } + } + } + + override fun forEachChild( + category: PortabilityCategory, + parentKey: String, + block: (PortabilityRecord) -> Unit, + ) { + flush() + connection.prepareStatement( + "SELECT payload FROM records WHERE category = ? AND parent_key = ? ORDER BY ordinal", + ).use { statement -> + statement.setString(1, category.wireName) + statement.setString(2, portabilityStableHash(parentKey.trim().lowercase())) + statement.executeQuery().use { rows -> + while (rows.next()) block(CacheJson.decodeFromString(rows.getString(1))) + } + } + } + + fun flush() { + checkOpen() + if (pendingWrites == 0) return + connection.commit() + pendingWrites = 0 + } + + override fun close() { + if (closed) return + runCatching { flush() } + insertRecord.close() + markCategory.close() + upsertIssue.close() + putLookup.close() + getLookup.close() + connection.close() + closed = true + } + + fun delete() { + close() + Files.deleteIfExists(path) + Files.deleteIfExists(path.resolveSibling("${path.fileName}-wal")) + Files.deleteIfExists(path.resolveSibling("${path.fileName}-shm")) + } + + private fun flushWhenNeeded() { + if (pendingWrites >= COMMIT_INTERVAL) flush() + } + + private fun checkOpen() = check(!closed) { "Portability spool is closed" } + + companion object { + private const val COMMIT_INTERVAL = 500 + + fun create(directory: Path): PortabilitySpool = PortabilitySpoolSchema.create(directory) { path, connection -> + PortabilitySpool(path, connection) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt new file mode 100644 index 00000000..de0ff23d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.portability + +import java.security.MessageDigest +import java.util.HexFormat + +internal fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") + .digest(value.toByteArray(Charsets.UTF_8)) + .let(HexFormat.of()::formatHex) + +internal fun portabilityCategoryByWireName(value: String): PortabilityCategory = + PortabilityCategory.entries.firstOrNull { it.wireName == value } + ?: error("Unknown portability category: $value") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt new file mode 100644 index 00000000..7f81f08e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt @@ -0,0 +1,50 @@ +package dev.typetype.server.portability + +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.DriverManager + +internal object PortabilitySpoolSchema { + fun create(directory: Path, factory: (Path, Connection) -> T): T { + Files.createDirectories(directory) + val path = Files.createTempFile(directory, "portability-", ".sqlite") + val connection = DriverManager.getConnection("jdbc:sqlite:$path") + connection.createStatement().use { statement -> + statement.execute("PRAGMA journal_mode = WAL") + statement.execute("PRAGMA synchronous = NORMAL") + statement.execute("PRAGMA temp_store = MEMORY") + statement.execute( + """ + CREATE TABLE records( + ordinal INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + stable_key TEXT NOT NULL, + parent_key TEXT, + payload TEXT NOT NULL, + UNIQUE(category, stable_key) + ) + """.trimIndent(), + ) + statement.execute("CREATE INDEX records_parent_idx ON records(category, parent_key, ordinal)") + statement.execute("CREATE TABLE categories(category TEXT PRIMARY KEY)") + statement.execute( + "CREATE TABLE lookups(namespace TEXT NOT NULL, lookup_key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(namespace, lookup_key))", + ) + statement.execute( + """ + CREATE TABLE issues( + category TEXT NOT NULL, + code TEXT NOT NULL, + message TEXT NOT NULL, + item_count INTEGER NOT NULL, + UNIQUE(category, code, message) + ) + """.trimIndent(), + ) + } + connection.autoCommit = false + connection.commit() + return factory(path, connection) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt new file mode 100644 index 00000000..624b0699 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.portability + +import io.ktor.utils.io.ByteReadChannel +import io.ktor.utils.io.jvm.javaio.toInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path + +object PortabilityUploadWriter { + suspend fun write(channel: ByteReadChannel, target: Path): Long = withContext(Dispatchers.IO) { + var written = 0L + channel.toInputStream().use { input -> + Files.newOutputStream(target).use { output -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read <= 0) break + written += read + if (written > PortabilityLimits.MAX_UPLOAD_BYTES) throw PortabilityUploadTooLargeException() + output.write(buffer, 0, read) + } + } + } + written + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt new file mode 100644 index 00000000..c71f3a18 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.Serializable + +@Serializable +data class PortabilityVideo( + val url: String, + val title: String = "", + val thumbnailUrl: String = "", + val durationSeconds: Long = 0L, + val channelName: String = "", + val channelUrl: String = "", + val channelAvatarUrl: String = "", + val viewCount: Long = 0L, + val publishedAt: Long = -1L, +) + +internal fun PortabilityVideo.normalized(): PortabilityVideo = copy( + url = url.trim(), + title = title.trim(), + thumbnailUrl = thumbnailUrl.trim(), + durationSeconds = durationSeconds.coerceAtLeast(0L), + channelName = channelName.trim(), + channelUrl = channelUrl.trim(), + channelAvatarUrl = channelAvatarUrl.trim(), + viewCount = viewCount.coerceAtLeast(0L), +) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt new file mode 100644 index 00000000..5d73c058 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT +import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION +import java.io.OutputStream + +class TypeTypePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.TYPE_TYPE, + adapterVersion = 1, + capabilities = PortabilityCategory.entries.mapTo(linkedSetOf()) { category -> + PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.COMPLETE, + ) + }, + defaultExtension = "json", + contentType = "application/json", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + if (input.archive != null) return null + val probe = input.probe.decodeToString() + val formatMarker = Regex("\"format\"\\s*:\\s*\"${Regex.escape(TYPE_TYPE_BACKUP_FORMAT)}\"") + if (!formatMarker.containsMatchIn(probe)) return null + val version = Regex("\"version\"\\s*:\\s*(\\d+)").find(probe)?.groupValues?.get(1) + return PortabilityDetection(PortabilityFormat.TYPE_TYPE, version, 100, "TypeType backup marker") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) = input.withJsonParser { parser -> + parser.requireObject() + var format: String? = null + var version: Int? = null + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "format" -> format = parser.textOrEmpty() + "version" -> version = parser.intValue + "subscriptions" -> sink.read(PortabilityCategory.SUBSCRIPTIONS) { TypeTypePortabilityReader.subscriptions(parser, token, sink) } + "subscriptionGroups" -> sink.read(PortabilityCategory.SUBSCRIPTION_GROUPS) { TypeTypePortabilityReader.groups(parser, token, sink) } + "history" -> sink.read(PortabilityCategory.HISTORY) { TypeTypePortabilityReader.history(parser, token, sink) } + "playlists" -> sink.read(PortabilityCategory.PLAYLISTS) { TypeTypePortabilityReader.playlists(parser, token, sink) } + "watchLater" -> sink.read(PortabilityCategory.WATCH_LATER) { TypeTypePortabilityReader.watchLater(parser, token, sink) } + "favorites" -> sink.read(PortabilityCategory.FAVORITES) { TypeTypePortabilityReader.favorites(parser, token, sink) } + "progress" -> sink.read(PortabilityCategory.PROGRESS) { TypeTypePortabilityReader.progress(parser, token, sink) } + "searchHistory" -> sink.read(PortabilityCategory.SEARCH_HISTORY) { TypeTypePortabilityReader.searchHistory(parser, token, sink) } + "savedPlaylists" -> sink.read(PortabilityCategory.SAVED_PLAYLISTS) { TypeTypePortabilityReader.savedPlaylists(parser, token, sink) } + "settings" -> sink.read(PortabilityCategory.SETTINGS) { TypeTypePortabilityReader.settings(parser, token, sink) } + "contentFilters" -> sink.read(PortabilityCategory.CONTENT_FILTERS) { TypeTypePortabilityReader.contentFilters(parser, token, sink) } + else -> parser.skipChildren() + } + } + require(format == TYPE_TYPE_BACKUP_FORMAT) { "Unsupported TypeType backup format" } + require(version == TYPE_TYPE_BACKUP_VERSION) { "Unsupported TypeType backup version" } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) = TypeTypePortabilityWriter.write(source, output, categories) +} + +private inline fun PortabilityRecordSink.read(category: PortabilityCategory, block: () -> Unit) { + markCategory(category) + block() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt new file mode 100644 index 00000000..08c313c3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt @@ -0,0 +1,124 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityCoreExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, sink) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, sink) + PortabilityCategory.HISTORY -> history(userId, sink) + PortabilityCategory.PLAYLISTS -> playlists(userId, sink) + else -> error("Unsupported core portability category") + } + } + + private fun subscriptions(userId: String, sink: PortabilityRecordSink) { + SubscriptionsTable.selectAll().where { SubscriptionsTable.userId eq userId } + .orderBy(SubscriptionsTable.subscribedAt, SortOrder.ASC) + .forEach { row -> + sink.write( + PortabilitySubscription( + row[SubscriptionsTable.channelUrl], + row[SubscriptionsTable.name], + row[SubscriptionsTable.avatarUrl], + row[SubscriptionsTable.subscribedAt], + ), + ) + } + } + + private fun groups(userId: String, sink: PortabilityRecordSink) { + val groups = SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt, SortOrder.ASC) + groups.forEach { row -> + val groupId = row[SubscriptionGroupsTable.id] + val name = row[SubscriptionGroupsTable.name] + sink.write(PortabilitySubscriptionGroup(name)) + SubscriptionGroupMembershipsTable.selectAll().where { + SubscriptionGroupMembershipsTable.groupId eq groupId + }.orderBy(SubscriptionGroupMembershipsTable.addedAt, SortOrder.ASC).forEach { membership -> + sink.write( + PortabilitySubscriptionGroupMembership( + name, + membership[SubscriptionGroupMembershipsTable.channelUrl], + ), + ) + } + } + } + + private fun history(userId: String, sink: PortabilityRecordSink) { + HistoryTable.selectAll().where { HistoryTable.userId eq userId } + .orderBy(HistoryTable.watchedAt, SortOrder.ASC) + .forEach { row -> + sink.write( + PortabilityHistory( + video = PortabilityVideo( + url = row[HistoryTable.url], + title = row[HistoryTable.title], + thumbnailUrl = row[HistoryTable.thumbnail], + durationSeconds = row[HistoryTable.duration], + channelName = row[HistoryTable.channelName], + channelUrl = row[HistoryTable.channelUrl], + channelAvatarUrl = row[HistoryTable.channelAvatar], + ), + watchedAt = row[HistoryTable.watchedAt], + positionSeconds = row[HistoryTable.progress], + ), + ) + } + } + + private fun playlists(userId: String, sink: PortabilityRecordSink) { + PlaylistsTable.selectAll().where { PlaylistsTable.userId eq userId } + .orderBy(PlaylistsTable.createdAt, SortOrder.ASC) + .forEach { row -> + val id = row[PlaylistsTable.id] + sink.write( + PortabilityPlaylist( + id, + row[PlaylistsTable.name], + row[PlaylistsTable.description], + row[PlaylistsTable.createdAt], + ), + ) + playlistVideos(userId, id, sink) + } + } + + private fun playlistVideos(userId: String, playlistId: String, sink: PortabilityRecordSink) { + PlaylistVideosTable.selectAll().where { + (PlaylistVideosTable.userId eq userId) and (PlaylistVideosTable.playlistId eq playlistId) + }.orderBy(PlaylistVideosTable.position, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityPlaylistVideo( + playlistId, + row[PlaylistVideosTable.position], + PortabilityVideo( + row[PlaylistVideosTable.url], + row[PlaylistVideosTable.title], + row[PlaylistVideosTable.thumbnail], + row[PlaylistVideosTable.duration], + row[PlaylistVideosTable.channelName], + row[PlaylistVideosTable.channelUrl], + row[PlaylistVideosTable.channelAvatar], + row[PlaylistVideosTable.viewCount], + row[PlaylistVideosTable.publishedAt], + ), + row[PlaylistVideosTable.addedAt], + ), + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt new file mode 100644 index 00000000..5f867ea3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt @@ -0,0 +1,185 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.PlaylistsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.services.ChannelUrlCanonicalizer +import dev.typetype.server.services.SubscriptionMutationLock +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.nio.charset.StandardCharsets +import java.util.Locale +import java.util.UUID + +internal object TypeTypePortabilityCoreImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long = when (category) { + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy) + PortabilityCategory.HISTORY -> history(userId, source, policy) + PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy) + else -> error("Unsupported core portability category") + } + + private fun subscriptions( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + SubscriptionMutationLock.acquire(userId) + if (policy == PortabilityDuplicatePolicy.REPLACE) { + SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + if (record !is PortabilitySubscription) return@forEach + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + count += SubscriptionsTable.insertIgnore { + it[SubscriptionsTable.userId] = userId + it[SubscriptionsTable.channelUrl] = channelUrl + it[SubscriptionsTable.name] = record.name + it[SubscriptionsTable.avatarUrl] = record.avatarUrl + it[SubscriptionsTable.subscribedAt] = record.subscribedAt + }.insertedCount + } + return count + } + + private fun groups( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + SubscriptionMutationLock.acquire(userId) + if (policy == PortabilityDuplicatePolicy.REPLACE) { + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + val normalized = record.name.trim().lowercase(Locale.ROOT) + require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } + count += SubscriptionGroupsTable.insertIgnore { + it[id] = stableId(userId, "group:$normalized") + it[SubscriptionGroupsTable.userId] = userId + it[name] = record.name.trim() + it[normalizedName] = normalized + it[createdAt] = 0L + it[updatedAt] = 0L + }.insertedCount + } + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroupMembership) return@forEach + val normalized = record.groupName.trim().lowercase(Locale.ROOT) + val groupId = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalized) + }.singleOrNull()?.get(SubscriptionGroupsTable.id) ?: return@forEach + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.empty().not() + if (!subscribed) return@forEach + count += SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = 0L + }.insertedCount + } + return count + } + + private fun history( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) HistoryTable.deleteWhere { HistoryTable.userId eq userId } + var count = 0L + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record !is PortabilityHistory) return@forEach + val exists = HistoryTable.selectAll().where { + (HistoryTable.userId eq userId) and + (HistoryTable.url eq record.video.url) and + (HistoryTable.watchedAt eq record.watchedAt) + }.empty().not() + if (!exists) { + count += HistoryTable.insertIgnore { + it[id] = UUID.randomUUID().toString() + it[HistoryTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[duration] = record.video.durationSeconds + it[progress] = record.positionSeconds + it[watchedAt] = record.watchedAt + }.insertedCount + } + } + return count + } + + private fun playlists( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) { + PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } + PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } + } + var count = 0L + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record is PortabilityPlaylist) count += insertPlaylist(userId, record) + } + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record is PortabilityPlaylistVideo) count += insertPlaylistVideo(userId, record) + } + return count + } + + private fun insertPlaylist(userId: String, record: PortabilityPlaylist): Int = PlaylistsTable.insertIgnore { + it[id] = stableId(userId, "playlist:${record.sourceId}") + it[PlaylistsTable.userId] = userId + it[name] = record.name + it[description] = record.description + it[createdAt] = record.createdAt + }.insertedCount + + private fun insertPlaylistVideo(userId: String, record: PortabilityPlaylistVideo): Int = + PlaylistVideosTable.insertIgnore { + it[id] = stableId(userId, "playlist:${record.playlistSourceId}:${record.position}:${record.video.url}") + it[playlistId] = stableId(userId, "playlist:${record.playlistSourceId}") + it[PlaylistVideosTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[position] = record.position + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[addedAt] = record.addedAt + it[publishedAt] = record.video.publishedAt + }.insertedCount +} + +private fun stableId(userId: String, value: String): String = UUID.nameUUIDFromBytes( + "$userId:$value".toByteArray(StandardCharsets.UTF_8), +).toString() diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt new file mode 100644 index 00000000..6d133576 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt @@ -0,0 +1,121 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityCoreWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + subscriptions(json, source, categories) + groups(json, source, categories) + history(json, source, categories) + playlists(json, source, categories) + } + + private fun subscriptions( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SUBSCRIPTIONS !in categories && PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptions") + source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> + if (record !is PortabilitySubscription) return@forEach + json.writeStartObject() + json.writeStringField("channelUrl", record.channelUrl) + json.writeStringField("name", record.name) + json.writeStringField("avatarUrl", record.avatarUrl) + json.writeNumberField("subscribedAt", record.subscribedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun groups( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SUBSCRIPTION_GROUPS !in categories) return + json.writeArrayFieldStart("subscriptionGroups") + source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> + if (record !is PortabilitySubscriptionGroup) return@forEach + json.writeStartObject() + json.writeStringField("name", record.name) + json.writeArrayFieldStart("channelUrls") + source.forEachChild(PortabilityCategory.SUBSCRIPTION_GROUPS, record.name) { child -> + if (child is PortabilitySubscriptionGroupMembership) json.writeString(child.channelUrl) + } + json.writeEndArray() + json.writeNumberField("createdAt", 0L) + json.writeNumberField("updatedAt", 0L) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun history( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.HISTORY !in categories) return + json.writeArrayFieldStart("history") + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record !is PortabilityHistory) return@forEach + json.writeStartObject() + json.writeStringField("url", record.video.url) + json.writeVideoFields(record.video) + json.writeNumberField("progress", record.positionSeconds) + json.writeNumberField("watchedAt", record.watchedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun playlists( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.PLAYLISTS !in categories) return + json.writeArrayFieldStart("playlists") + source.forEach(PortabilityCategory.PLAYLISTS) { record -> + if (record !is PortabilityPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("id", record.sourceId) + json.writeStringField("name", record.name) + json.writeStringField("description", record.description) + json.writeArrayFieldStart("videos") + var count = 0 + source.forEachChild(PortabilityCategory.PLAYLISTS, record.sourceId) { child -> + if (child !is PortabilityPlaylistVideo) return@forEachChild + json.writeStartObject() + json.writeStringField("url", child.video.url) + json.writeVideoFields(child.video) + json.writeNumberField("position", child.position) + json.writeNumberField("addedAt", child.addedAt) + json.writeEndObject() + count += 1 + } + json.writeEndArray() + json.writeNumberField("videoCount", count) + json.writeNumberField("createdAt", record.createdAt) + json.writeEndObject() + } + json.writeEndArray() + } +} + +internal fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { + writeStringField("title", video.title) + writeStringField("thumbnail", video.thumbnailUrl) + writeNumberField("duration", video.durationSeconds) + writeStringField("channelName", video.channelName) + writeStringField("channelUrl", video.channelUrl) + writeStringField("channelAvatar", video.channelAvatarUrl) + writeNumberField("viewCount", video.viewCount) + writeNumberField("publishedAt", video.publishedAt) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt new file mode 100644 index 00000000..343a103b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionFeedCacheInvalidation + +class TypeTypePortabilityDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ): Map { + val result = linkedMapOf() + request.categories.sortedBy(PortabilityCategory::wireName).forEach { category -> + val imported = DatabaseFactory.query { + TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) + } + result[category.wireName] = imported + onCategoryComplete(category, imported) + } + if (PortabilityCategory.SUBSCRIPTIONS in request.categories) { + SubscriptionFeedCacheInvalidation.invalidate(userId) + } + return result + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + categories.sortedBy(PortabilityCategory::wireName).forEach { category -> + sink.markCategory(category) + DatabaseFactory.query { TypeTypePortabilityExport.write(userId, category, sink) } + onCategoryComplete(category, sinkCount(sink, category)) + } + } +} + +private fun sinkCount(sink: PortabilityRecordSink, category: PortabilityCategory): Long = when (sink) { + is ProgressRecordSink -> sink.count(category) + is PortabilityRecordSource -> sink.counts()[category] ?: 0L + else -> 0L +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt new file mode 100644 index 00000000..7078efd8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt @@ -0,0 +1,21 @@ +package dev.typetype.server.portability + +internal object TypeTypePortabilityExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + -> TypeTypePortabilityCoreExport.write(userId, category, sink) + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, + -> TypeTypePortabilityLibraryExport.write(userId, category, sink) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsExport.write(userId, sink) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterExport.write(userId, sink) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt new file mode 100644 index 00000000..a575fa7b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityFilterExport { + private const val USER_SCOPE = "user" + + fun write(userId: String, sink: PortabilityRecordSink) { + blockedChannels(userId, sink) + blockedVideos(userId, sink) + blockedKeywords(userId, sink) + allowedChannels(userId, sink) + allowedPlaylists(userId, sink) + } + + private fun blockedChannels(userId: String, sink: PortabilityRecordSink) { + BlockedChannelsTable.selectAll().where { + (BlockedChannelsTable.userId eq userId) and (BlockedChannelsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedChannel", + row[BlockedChannelsTable.channelUrl], + row[BlockedChannelsTable.channelName].orEmpty(), + row[BlockedChannelsTable.channelThumbnailUrl].orEmpty(), + row[BlockedChannelsTable.blockedAt], + ), + ) + } + } + + private fun blockedVideos(userId: String, sink: PortabilityRecordSink) { + BlockedVideosTable.selectAll().where { + (BlockedVideosTable.userId eq userId) and (BlockedVideosTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedVideo", + row[BlockedVideosTable.videoUrl], + createdAt = row[BlockedVideosTable.blockedAt], + ), + ) + } + } + + private fun blockedKeywords(userId: String, sink: PortabilityRecordSink) { + BlockedKeywordsTable.selectAll().where { + (BlockedKeywordsTable.userId eq userId) and (BlockedKeywordsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "blockedKeyword", + row[BlockedKeywordsTable.keyword], + createdAt = row[BlockedKeywordsTable.blockedAt], + ), + ) + } + } + + private fun allowedChannels(userId: String, sink: PortabilityRecordSink) { + AllowedChannelsTable.selectAll().where { + (AllowedChannelsTable.userId eq userId) and (AllowedChannelsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "allowedChannel", + row[AllowedChannelsTable.channelUrl], + row[AllowedChannelsTable.channelName].orEmpty(), + row[AllowedChannelsTable.channelThumbnailUrl].orEmpty(), + row[AllowedChannelsTable.allowedAt], + ), + ) + } + } + + private fun allowedPlaylists(userId: String, sink: PortabilityRecordSink) { + AllowedPlaylistsTable.selectAll().where { + (AllowedPlaylistsTable.userId eq userId) and (AllowedPlaylistsTable.scope eq USER_SCOPE) + }.forEach { row -> + sink.write( + PortabilityContentFilter( + "allowedPlaylist", + row[AllowedPlaylistsTable.playlistUrl], + row[AllowedPlaylistsTable.title].orEmpty(), + row[AllowedPlaylistsTable.thumbnailUrl].orEmpty(), + row[AllowedPlaylistsTable.allowedAt], + buildJsonObject { + put("uploaderName", row[AllowedPlaylistsTable.uploaderName].orEmpty()) + }, + ), + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt new file mode 100644 index 00000000..819dcc67 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.AllowedChannelsTable +import dev.typetype.server.db.tables.AllowedPlaylistsTable +import dev.typetype.server.db.tables.BlockedChannelsTable +import dev.typetype.server.db.tables.BlockedKeywordsTable +import dev.typetype.server.db.tables.BlockedVideosTable +import dev.typetype.server.services.normalizeBlockedKeyword +import dev.typetype.server.services.normalizeChannelKey +import dev.typetype.server.services.normalizePlaylistKey +import kotlinx.serialization.json.jsonPrimitive +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityFilterImport { + private const val USER_SCOPE = "user" + + fun write( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) clear(userId) + var count = 0L + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + if (record is PortabilityContentFilter) count += insert(userId, record) + } + return count + } + + private fun insert(userId: String, record: PortabilityContentFilter): Int = when (record.kind) { + "blockedChannel" -> BlockedChannelsTable.insertIgnore { + it[BlockedChannelsTable.userId] = userId + it[scope] = USER_SCOPE + it[channelUrl] = record.value + it[channelName] = record.label + it[channelThumbnailUrl] = record.imageUrl + it[blockedAt] = record.createdAt + }.insertedCount + "blockedVideo" -> BlockedVideosTable.insertIgnore { + it[BlockedVideosTable.userId] = userId + it[scope] = USER_SCOPE + it[videoUrl] = record.value + it[blockedAt] = record.createdAt + }.insertedCount + "blockedKeyword" -> BlockedKeywordsTable.insertIgnore { + it[BlockedKeywordsTable.userId] = userId + it[scope] = USER_SCOPE + it[keyword] = normalizeBlockedKeyword(record.value) + it[blockedAt] = record.createdAt + }.insertedCount + "allowedChannel" -> AllowedChannelsTable.insertIgnore { + it[AllowedChannelsTable.userId] = userId + it[scope] = USER_SCOPE + it[channelUrl] = normalizeChannelKey(record.value) + it[channelName] = record.label + it[channelThumbnailUrl] = record.imageUrl + it[allowedAt] = record.createdAt + }.insertedCount + "allowedPlaylist" -> allowedPlaylist(userId, record) + else -> throw IllegalArgumentException("Unsupported content filter kind") + } + + private fun allowedPlaylist(userId: String, record: PortabilityContentFilter): Int { + val url = normalizePlaylistKey(record.value) + val exists = AllowedPlaylistsTable.selectAll().where { + (AllowedPlaylistsTable.userId eq userId) and + (AllowedPlaylistsTable.scope eq USER_SCOPE) and + (AllowedPlaylistsTable.playlistUrl eq url) + }.empty().not() + if (exists) return 0 + return AllowedPlaylistsTable.insertIgnore { + it[AllowedPlaylistsTable.userId] = userId + it[scope] = USER_SCOPE + it[playlistUrl] = url + it[title] = record.label + it[thumbnailUrl] = record.imageUrl + it[uploaderName] = record.metadata["uploaderName"]?.jsonPrimitive?.content.orEmpty() + it[allowedAt] = record.createdAt + }.insertedCount + } + + private fun clear(userId: String) { + BlockedChannelsTable.deleteWhere { + (BlockedChannelsTable.userId eq userId) and (BlockedChannelsTable.scope eq USER_SCOPE) + } + BlockedVideosTable.deleteWhere { + (BlockedVideosTable.userId eq userId) and (BlockedVideosTable.scope eq USER_SCOPE) + } + BlockedKeywordsTable.deleteWhere { + (BlockedKeywordsTable.userId eq userId) and (BlockedKeywordsTable.scope eq USER_SCOPE) + } + AllowedChannelsTable.deleteWhere { + (AllowedChannelsTable.userId eq userId) and (AllowedChannelsTable.scope eq USER_SCOPE) + } + AllowedPlaylistsTable.deleteWhere { + (AllowedPlaylistsTable.userId eq userId) and (AllowedPlaylistsTable.scope eq USER_SCOPE) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt new file mode 100644 index 00000000..5e37bb41 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityFilterWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.CONTENT_FILTERS !in categories) return + json.writeObjectFieldStart("contentFilters") + writeArray(json, source, "blockedChannels", "blockedChannel") + writeArray(json, source, "blockedVideos", "blockedVideo") + writeArray(json, source, "blockedKeywords", "blockedKeyword") + writeArray(json, source, "allowedChannels", "allowedChannel") + writeArray(json, source, "allowedPlaylists", "allowedPlaylist") + json.writeEndObject() + } + + private fun writeArray( + json: JsonGenerator, + source: PortabilityRecordSource, + field: String, + kind: String, + ) { + json.writeArrayFieldStart(field) + source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> + if (record !is PortabilityContentFilter || record.kind != kind) return@forEach + json.writeStartObject() + when (kind) { + "blockedKeyword" -> json.writeStringField("keyword", record.value) + else -> json.writeStringField("url", record.value) + } + if (kind == "allowedPlaylist") { + json.writeStringField("title", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeStringField("uploaderName", record.metadata["uploaderName"]?.toString()?.trim('"').orEmpty()) + json.writeNumberField("allowedAt", record.createdAt) + } else if (kind.startsWith("allowed")) { + json.writeStringField("name", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeNumberField("allowedAt", record.createdAt) + } else { + json.writeStringField("name", record.label) + json.writeStringField("thumbnailUrl", record.imageUrl) + json.writeNumberField("blockedAt", record.createdAt) + } + json.writeEndObject() + } + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt new file mode 100644 index 00000000..1fa94f38 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.portability + +internal object TypeTypePortabilityImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long = when (category) { + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.SUBSCRIPTION_GROUPS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + -> TypeTypePortabilityCoreImport.write(userId, category, source, policy) + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, + PortabilityCategory.PROGRESS, + PortabilityCategory.SEARCH_HISTORY, + PortabilityCategory.SAVED_PLAYLISTS, + -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt new file mode 100644 index 00000000..fb7d4dd3 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.WatchLaterTable +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilityLibraryExport { + fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { + when (category) { + PortabilityCategory.WATCH_LATER -> watchLater(userId, sink) + PortabilityCategory.FAVORITES -> favorites(userId, sink) + PortabilityCategory.PROGRESS -> progress(userId, sink) + PortabilityCategory.SEARCH_HISTORY -> searchHistory(userId, sink) + PortabilityCategory.SAVED_PLAYLISTS -> savedPlaylists(userId, sink) + else -> error("Unsupported library portability category") + } + } + + private fun watchLater(userId: String, sink: PortabilityRecordSink) { + WatchLaterTable.selectAll().where { WatchLaterTable.userId eq userId } + .orderBy(WatchLaterTable.addedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityWatchLater( + PortabilityVideo( + row[WatchLaterTable.url], + row[WatchLaterTable.title], + row[WatchLaterTable.thumbnail], + row[WatchLaterTable.duration], + row[WatchLaterTable.channelName], + row[WatchLaterTable.channelUrl], + row[WatchLaterTable.channelAvatar], + row[WatchLaterTable.viewCount], + row[WatchLaterTable.publishedAt], + ), + row[WatchLaterTable.addedAt], + ), + ) + } + } + + private fun favorites(userId: String, sink: PortabilityRecordSink) { + FavoritesTable.selectAll().where { FavoritesTable.userId eq userId } + .orderBy(FavoritesTable.favoritedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityFavorite( + PortabilityVideo( + row[FavoritesTable.videoUrl], + row[FavoritesTable.title], + row[FavoritesTable.thumbnail], + row[FavoritesTable.duration], + row[FavoritesTable.channelName], + row[FavoritesTable.channelUrl], + row[FavoritesTable.channelAvatar], + row[FavoritesTable.viewCount], + row[FavoritesTable.publishedAt], + ), + row[FavoritesTable.favoritedAt], + ), + ) + } + } + + private fun progress(userId: String, sink: PortabilityRecordSink) { + ProgressTable.selectAll().where { ProgressTable.userId eq userId } + .orderBy(ProgressTable.updatedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilityProgress( + row[ProgressTable.videoUrl], + row[ProgressTable.position], + row[ProgressTable.updatedAt], + ), + ) + } + } + + private fun searchHistory(userId: String, sink: PortabilityRecordSink) { + SearchHistoryTable.selectAll().where { SearchHistoryTable.userId eq userId } + .orderBy(SearchHistoryTable.searchedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilitySearchHistory( + row[SearchHistoryTable.term], + row[SearchHistoryTable.searchedAt], + ), + ) + } + } + + private fun savedPlaylists(userId: String, sink: PortabilityRecordSink) { + SavedPlaylistsTable.selectAll().where { SavedPlaylistsTable.userId eq userId } + .orderBy(SavedPlaylistsTable.savedAt, SortOrder.ASC).forEach { row -> + sink.write( + PortabilitySavedPlaylist( + row[SavedPlaylistsTable.publicPlaylistId], + row[SavedPlaylistsTable.url], + row[SavedPlaylistsTable.title], + row[SavedPlaylistsTable.thumbnailUrl], + row[SavedPlaylistsTable.uploaderName], + row[SavedPlaylistsTable.streamCount], + row[SavedPlaylistsTable.playlistType], + row[SavedPlaylistsTable.savedAt], + ), + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt new file mode 100644 index 00000000..d1b3ae65 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt @@ -0,0 +1,121 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SavedPlaylistsTable +import dev.typetype.server.db.tables.SearchHistoryTable +import dev.typetype.server.db.tables.WatchLaterTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.nio.charset.StandardCharsets +import java.util.UUID + +internal object TypeTypePortabilityLibraryImport { + fun write( + userId: String, + category: PortabilityCategory, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + ): Long { + clearIfReplacing(userId, category, policy) + var count = 0L + source.forEach(category) { record -> + count += when (record) { + is PortabilityWatchLater -> watchLater(userId, record) + is PortabilityFavorite -> favorite(userId, record) + is PortabilityProgress -> progress(userId, record) + is PortabilitySearchHistory -> searchHistory(userId, record) + is PortabilitySavedPlaylist -> savedPlaylist(userId, record) + else -> 0 + } + } + return count + } + + private fun clearIfReplacing( + userId: String, + category: PortabilityCategory, + policy: PortabilityDuplicatePolicy, + ) { + if (policy != PortabilityDuplicatePolicy.REPLACE) return + when (category) { + PortabilityCategory.WATCH_LATER -> WatchLaterTable.deleteWhere { WatchLaterTable.userId eq userId } + PortabilityCategory.FAVORITES -> FavoritesTable.deleteWhere { FavoritesTable.userId eq userId } + PortabilityCategory.PROGRESS -> ProgressTable.deleteWhere { ProgressTable.userId eq userId } + PortabilityCategory.SEARCH_HISTORY -> SearchHistoryTable.deleteWhere { SearchHistoryTable.userId eq userId } + PortabilityCategory.SAVED_PLAYLISTS -> SavedPlaylistsTable.deleteWhere { SavedPlaylistsTable.userId eq userId } + else -> error("Unsupported library portability category") + } + } + + private fun watchLater(userId: String, record: PortabilityWatchLater): Int = WatchLaterTable.insertIgnore { + it[WatchLaterTable.userId] = userId + it[url] = record.video.url + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[addedAt] = record.addedAt + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[publishedAt] = record.video.publishedAt + }.insertedCount + + private fun favorite(userId: String, record: PortabilityFavorite): Int = FavoritesTable.insertIgnore { + it[FavoritesTable.userId] = userId + it[videoUrl] = record.video.url + it[favoritedAt] = record.favoritedAt + it[title] = record.video.title + it[thumbnail] = record.video.thumbnailUrl + it[duration] = record.video.durationSeconds + it[channelName] = record.video.channelName + it[channelUrl] = record.video.channelUrl + it[channelAvatar] = record.video.channelAvatarUrl + it[viewCount] = record.video.viewCount + it[publishedAt] = record.video.publishedAt + }.insertedCount + + private fun progress(userId: String, record: PortabilityProgress): Int = ProgressTable.insertIgnore { + it[ProgressTable.userId] = userId + it[videoUrl] = record.videoUrl + it[position] = record.positionSeconds.coerceAtLeast(0L) + it[updatedAt] = record.updatedAt + }.insertedCount + + private fun searchHistory(userId: String, record: PortabilitySearchHistory): Int { + val exists = SearchHistoryTable.selectAll().where { + (SearchHistoryTable.userId eq userId) and + (SearchHistoryTable.term eq record.term) and + (SearchHistoryTable.searchedAt eq record.searchedAt) + }.empty().not() + if (exists) return 0 + return SearchHistoryTable.insertIgnore { + it[id] = UUID.randomUUID().toString() + it[SearchHistoryTable.userId] = userId + it[term] = record.term + it[searchedAt] = record.searchedAt + }.insertedCount + } + + private fun savedPlaylist(userId: String, record: PortabilitySavedPlaylist): Int = + SavedPlaylistsTable.insertIgnore { + it[id] = stableLibraryId(userId, record.url) + it[SavedPlaylistsTable.userId] = userId + it[publicPlaylistId] = record.sourceId + it[url] = record.url + it[title] = record.title + it[thumbnailUrl] = record.thumbnailUrl + it[uploaderName] = record.uploaderName + it[streamCount] = record.streamCount + it[playlistType] = record.playlistType + it[savedAt] = record.savedAt + }.insertedCount +} + +private fun stableLibraryId(userId: String, value: String): String = UUID.nameUUIDFromBytes( + "$userId:saved:$value".toByteArray(StandardCharsets.UTF_8), +).toString() diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt new file mode 100644 index 00000000..31d2946f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt @@ -0,0 +1,105 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilityLibraryWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + writeVideos(json, source, categories, PortabilityCategory.WATCH_LATER, "watchLater") + writeVideos(json, source, categories, PortabilityCategory.FAVORITES, "favorites") + progress(json, source, categories) + searchHistory(json, source, categories) + savedPlaylists(json, source, categories) + } + + private fun writeVideos( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + category: PortabilityCategory, + field: String, + ) { + if (category !in categories) return + json.writeArrayFieldStart(field) + source.forEach(category) { record -> + val video = when (record) { + is PortabilityWatchLater -> record.video + is PortabilityFavorite -> record.video + else -> return@forEach + } + json.writeStartObject() + if (record is PortabilityFavorite) { + json.writeStringField("videoUrl", video.url) + json.writeNumberField("favoritedAt", record.favoritedAt) + } else { + json.writeStringField("url", video.url) + json.writeNumberField("addedAt", (record as PortabilityWatchLater).addedAt) + } + json.writeVideoFields(video) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun progress( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.PROGRESS !in categories) return + json.writeArrayFieldStart("progress") + source.forEach(PortabilityCategory.PROGRESS) { record -> + if (record !is PortabilityProgress) return@forEach + json.writeStartObject() + json.writeStringField("videoUrl", record.videoUrl) + json.writeNumberField("position", record.positionSeconds) + json.writeNumberField("updatedAt", record.updatedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun searchHistory( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SEARCH_HISTORY !in categories) return + json.writeArrayFieldStart("searchHistory") + source.forEach(PortabilityCategory.SEARCH_HISTORY) { record -> + if (record !is PortabilitySearchHistory) return@forEach + json.writeStartObject() + json.writeStringField("term", record.term) + json.writeNumberField("searchedAt", record.searchedAt) + json.writeEndObject() + } + json.writeEndArray() + } + + private fun savedPlaylists( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SAVED_PLAYLISTS !in categories) return + json.writeArrayFieldStart("savedPlaylists") + source.forEach(PortabilityCategory.SAVED_PLAYLISTS) { record -> + if (record !is PortabilitySavedPlaylist) return@forEach + json.writeStartObject() + json.writeStringField("id", record.sourceId) + json.writeStringField("publicPlaylistId", record.sourceId) + json.writeStringField("url", record.url) + json.writeStringField("title", record.title) + json.writeStringField("thumbnailUrl", record.thumbnailUrl) + json.writeStringField("uploaderName", record.uploaderName) + json.writeNumberField("streamCount", record.streamCount) + json.writeStringField("playlistType", record.playlistType) + json.writeNumberField("savedAt", record.savedAt) + json.writeEndObject() + } + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt new file mode 100644 index 00000000..a655b449 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt @@ -0,0 +1,159 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.AllowedChannelItem +import dev.typetype.server.models.AllowedPlaylistItem +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.models.ProgressItem +import dev.typetype.server.models.SavedPlaylistItem +import dev.typetype.server.models.SearchHistoryItem +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionGroupBackupItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.TypeTypeContentFiltersBackup +import dev.typetype.server.models.WatchLaterItem +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.put + +internal object TypeTypePortabilityReader { + fun subscriptions(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilitySubscription(it.channelUrl, it.name, it.avatarUrl, it.subscribedAt)) } + + fun groups(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { group -> + sink.write(PortabilitySubscriptionGroup(group.name)) + group.channelUrls.forEach { sink.write(PortabilitySubscriptionGroupMembership(group.name, it)) } + } + + fun history(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityHistory( + PortabilityVideo( + item.url, + item.title, + item.thumbnail, + item.duration, + item.channelName, + item.channelUrl, + item.channelAvatar, + ), + item.watchedAt, + item.progress, + ), + ) + } + + fun playlists(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { playlist -> + val sourceId = playlist.id.ifBlank { "${playlist.name}:${playlist.createdAt}" } + sink.write(PortabilityPlaylist(sourceId, playlist.name, playlist.description, playlist.createdAt)) + playlist.videos.forEach { video -> + sink.write( + PortabilityPlaylistVideo( + sourceId, + video.position, + PortabilityVideo( + video.url, + video.title, + video.thumbnail, + video.duration, + video.channelName, + video.channelUrl, + video.channelAvatar, + video.viewCount, + video.publishedAt, + ), + video.addedAt, + ), + ) + } + } + + fun watchLater(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityWatchLater( + PortabilityVideo(item.url, item.title, item.thumbnail, item.duration, item.channelName, item.channelUrl, item.channelAvatar, item.viewCount, item.publishedAt), + item.addedAt, + ), + ) + } + + fun favorites(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { item -> + sink.write( + PortabilityFavorite( + PortabilityVideo(item.videoUrl, item.title, item.thumbnail, item.duration, item.channelName, item.channelUrl, item.channelAvatar, item.viewCount, item.publishedAt), + item.favoritedAt, + ), + ) + } + + fun progress(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilityProgress(it.videoUrl, it.position, it.updatedAt)) } + + fun searchHistory(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { sink.write(PortabilitySearchHistory(it.term, it.searchedAt)) } + + fun savedPlaylists(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) = + parser.readArray(token) { + sink.write(PortabilitySavedPlaylist(it.publicPlaylistId, it.url, it.title, it.thumbnailUrl, it.uploaderName, it.streamCount, it.playlistType, it.savedAt)) + } + + fun settings(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) { + if (token == JsonToken.VALUE_NULL) return + val item = parser.decodeCurrent() + sink.write(PortabilitySettings(CacheJson.parseToJsonElement(CacheJson.encodeToString(SettingsItem.serializer(), item)).jsonObject)) + } + + fun contentFilters(parser: JsonParser, token: JsonToken, sink: PortabilityRecordSink) { + if (token == JsonToken.VALUE_NULL) return + val filters = parser.decodeCurrent() + filters.blockedChannels.forEach { sink.write(it.toFilter("blockedChannel")) } + filters.blockedVideos.forEach { sink.write(it.toFilter("blockedVideo")) } + filters.blockedKeywords.forEach { sink.write(PortabilityContentFilter("blockedKeyword", it.keyword, createdAt = it.blockedAt)) } + filters.allowedChannels.forEach { sink.write(it.toFilter()) } + filters.allowedPlaylists.forEach { sink.write(it.toFilter()) } + } +} + +private inline fun JsonParser.readArray(token: JsonToken, block: (T) -> Unit) { + if (token == JsonToken.VALUE_NULL) return + require(token == JsonToken.START_ARRAY) { "Invalid TypeType backup section" } + while (nextToken() != JsonToken.END_ARRAY) block(decodeCurrent()) +} + +private inline fun JsonParser.decodeCurrent(): T = + CacheJson.decodeFromString(readJsonElement().toString()) + +private fun BlockedItem.toFilter(kind: String) = PortabilityContentFilter( + kind, + url, + name.orEmpty(), + thumbnailUrl.orEmpty(), + blockedAt, +) + +private fun AllowedChannelItem.toFilter() = PortabilityContentFilter( + "allowedChannel", + url, + name.orEmpty(), + thumbnailUrl.orEmpty(), + allowedAt, +) + +private fun AllowedPlaylistItem.toFilter() = PortabilityContentFilter( + "allowedPlaylist", + url, + title.orEmpty(), + thumbnailUrl.orEmpty(), + allowedAt, + kotlinx.serialization.json.buildJsonObject { put("uploaderName", uploaderName.orEmpty()) }, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt new file mode 100644 index 00000000..f42d596b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.db.tables.SettingsTable +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.services.toSettingsItem +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.jsonObject +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object TypeTypePortabilitySettingsExport { + fun write(userId: String, sink: PortabilityRecordSink) { + val settings = SettingsTable.selectAll().where { SettingsTable.userId eq userId } + .singleOrNull()?.toSettingsItem() ?: SettingsItem() + val json = CacheJson.parseToJsonElement(CacheJson.encodeToString(settings)).jsonObject + sink.write(PortabilitySettings(json)) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt new file mode 100644 index 00000000..9479bf65 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.services.TypeTypeBackupLibraryRestore + +internal object TypeTypePortabilitySettingsImport { + fun write(userId: String, source: PortabilityRecordSource): Long { + var count = 0L + source.forEach(PortabilityCategory.SETTINGS) { record -> + if (record !is PortabilitySettings) return@forEach + val settings = CacheJson.decodeFromString(record.values.toString()) + count += TypeTypeBackupLibraryRestore.settings(userId, settings) + } + return count + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt new file mode 100644 index 00000000..0ca1d0f6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt @@ -0,0 +1,22 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator + +internal object TypeTypePortabilitySettingsWriter { + fun write( + json: JsonGenerator, + source: PortabilityRecordSource, + categories: Set, + ) { + if (PortabilityCategory.SETTINGS !in categories) return + json.writeFieldName("settings") + var written = false + source.forEach(PortabilityCategory.SETTINGS) { record -> + if (record is PortabilitySettings && !written) { + json.writeRawValue(record.values.toString()) + written = true + } + } + if (!written) json.writeNull() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt new file mode 100644 index 00000000..66fa7b75 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonGenerator +import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT +import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION +import java.io.OutputStream + +internal object TypeTypePortabilityWriter { + fun write( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + PortabilityJsonFactory.createGenerator(output).use { json -> + json.writeStartObject() + json.writeStringField("format", TYPE_TYPE_BACKUP_FORMAT) + json.writeNumberField("version", TYPE_TYPE_BACKUP_VERSION) + json.writeNumberField("exportedAt", System.currentTimeMillis()) + writeCategories(json, categories) + TypeTypePortabilityCoreWriter.write(json, source, categories) + TypeTypePortabilityLibraryWriter.write(json, source, categories) + TypeTypePortabilitySettingsWriter.write(json, source, categories) + TypeTypePortabilityFilterWriter.write(json, source, categories) + json.writeEndObject() + } + } + + private fun writeCategories(json: JsonGenerator, categories: Set) { + val legacy = categories.mapTo(linkedSetOf()) { + if (it == PortabilityCategory.SUBSCRIPTION_GROUPS) PortabilityCategory.SUBSCRIPTIONS.wireName else it.wireName + } + json.writeArrayFieldStart("categories") + legacy.sorted().forEach(json::writeString) + json.writeEndArray() + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt new file mode 100644 index 00000000..efb780da --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class ViewTubePortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.VIEW_TUBE, + 1, + setOf( + viewTubeCapability(PortabilityCategory.SUBSCRIPTIONS, PortabilityFidelity.COMPLETE), + viewTubeCapability(PortabilityCategory.HISTORY, PortabilityFidelity.COMPLETE), + viewTubeCapability(PortabilityCategory.PROGRESS, PortabilityFidelity.COMPLETE), + ), + "zip", + "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + if ("user.json" !in archive.names) return null + ZipFile(input.path.toFile()).use { zip -> + val entry = zip.getEntry("user.json") ?: return null + zip.getInputStream(entry).buffered().use { stream -> + val probe = stream.readNBytes(PortabilityLimits.PROBE_BYTES).decodeToString() + if (!probe.contains("\"username\"") || !probe.contains("\"subscriptions\"")) return null + if (!probe.contains("\"history\"") || !probe.contains("\"settings\"")) return null + } + } + return PortabilityDetection(PortabilityFormat.VIEW_TUBE, null, 99, "ViewTube user.json export") + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported ViewTube backup" } + ZipFile(input.path.toFile()).use { zip -> + val entry = requireNotNull(zip.getEntry("user.json")) + zip.getInputStream(entry).buffered().use { inputStream -> + PortabilityJsonFactory.createParser(inputStream).use { parser -> + ViewTubePortabilityReader.read(parser, sink) + } + } + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ): Unit = error("ViewTube does not provide a compatible full-backup import") +} + +private fun viewTubeCapability(category: PortabilityCategory, fidelity: PortabilityFidelity) = PortabilityCapability( + category, + setOf(PortabilityDirection.IMPORT), + fidelity, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt new file mode 100644 index 00000000..337594cc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt @@ -0,0 +1,160 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import java.time.Instant + +internal object ViewTubePortabilityReader { + fun read(parser: JsonParser, sink: PortabilityRecordSink) { + parser.requireObject() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val token = parser.nextToken() + when (field) { + "subscriptions" -> readContainer(parser, token, "channels") { readSubscription(parser, sink) } + "history" -> readContainer(parser, token, "videos") { readHistory(parser, sink) } + "settings" -> { + parser.skipChildren() + sink.issue(PortabilityIssue(PortabilityCategory.SETTINGS, "viewtube_settings_ignored", "ViewTube settings are not portable to TypeType")) + } + else -> parser.skipChildren() + } + } + } + + private fun readContainer(parser: JsonParser, token: JsonToken, arrayField: String, item: () -> Unit) { + if (token != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + val value = parser.nextToken() + if (field == arrayField && value == JsonToken.START_ARRAY) { + var count = 0 + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(++count <= PortabilityLimits.MAX_CONTAINER_RECORDS) { "$arrayField contains too many records" } + item() + } + } else { + parser.skipChildren() + } + } + } + + private fun readSubscription(parser: JsonParser, sink: PortabilityRecordSink) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + var id = "" + var name = "" + var url = "" + var avatar = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "authorId" -> id = parser.textOrEmpty() + "author" -> name = parser.textOrEmpty() + "authorUrl" -> url = parser.textOrEmpty() + "authorThumbnailUrl" -> avatar = parser.textOrEmpty() + "authorThumbnails" -> avatar = avatar.ifBlank { readFirstUrl(parser) } + else -> parser.skipChildren() + } + } + val channelUrl = url.ifBlank { youtubeChannelUrl(id) } + if (channelUrl.isNotBlank()) sink.write(PortabilitySubscription(channelUrl, name, avatar)) + } + + private fun readHistory(parser: JsonParser, sink: PortabilityRecordSink) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + val fields = ViewTubeHistoryFields() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "videoId" -> fields.videoId = parser.textOrEmpty() + "progressSeconds" -> fields.progress = parser.longOrZero() + "lengthSeconds" -> fields.duration = parser.longOrZero() + "lastVisit" -> fields.watchedAt = parseInstant(parser.textOrEmpty()) + "videoDetails" -> readVideoDetails(parser, fields) + else -> parser.skipChildren() + } + } + val url = youtubeVideoUrl(fields.videoId) + if (url.isBlank()) return + val video = PortabilityVideo( + url = url, + title = fields.title, + thumbnailUrl = fields.thumbnail, + durationSeconds = fields.duration, + channelName = fields.channelName, + channelUrl = youtubeChannelUrl(fields.channelId), + channelAvatarUrl = fields.channelAvatar, + viewCount = fields.views, + publishedAt = fields.publishedAt, + ) + sink.write(PortabilityHistory(video, fields.watchedAt, fields.progress)) + sink.write(PortabilityProgress(url, fields.progress, fields.watchedAt)) + } + + private fun readVideoDetails(parser: JsonParser, fields: ViewTubeHistoryFields) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + return + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + when (field) { + "videoId" -> fields.videoId = parser.textOrEmpty() + "title" -> fields.title = parser.textOrEmpty() + "author" -> fields.channelName = parser.textOrEmpty() + "authorId" -> fields.channelId = parser.textOrEmpty() + "authorThumbnailUrl" -> fields.channelAvatar = parser.textOrEmpty() + "lengthSeconds" -> fields.duration = parser.longOrZero() + "viewCount" -> fields.views = parser.longOrZero() + "published" -> fields.publishedAt = parser.longOrZero() + "videoThumbnails" -> fields.thumbnail = readFirstUrl(parser) + else -> parser.skipChildren() + } + } + } + + private fun readFirstUrl(parser: JsonParser): String { + if (parser.currentToken() != JsonToken.START_ARRAY) return parser.skipChildren().let { "" } + var url = "" + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName + parser.nextToken() + if (field == "url" && url.isBlank()) url = parser.textOrEmpty() else parser.skipChildren() + } + } + return url + } + + private fun parseInstant(value: String): Long = runCatching { Instant.parse(value).epochSecond }.getOrDefault(0L) +} + +private data class ViewTubeHistoryFields( + var videoId: String = "", + var progress: Long = 0, + var duration: Long = 0, + var watchedAt: Long = 0, + var title: String = "", + var thumbnail: String = "", + var channelName: String = "", + var channelId: String = "", + var channelAvatar: String = "", + var views: Long = 0, + var publishedAt: Long = -1, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt new file mode 100644 index 00000000..14577074 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -0,0 +1,135 @@ +package dev.typetype.server.portability + +import dev.typetype.server.models.PlaylistItem +import dev.typetype.server.services.YoutubeTakeoutCsvReader +import dev.typetype.server.services.YoutubeTakeoutRowParser +import dev.typetype.server.services.YoutubeTakeoutSchemaHints +import dev.typetype.server.services.YoutubeTakeoutSystemPlaylist +import java.io.BufferedReader +import java.io.InputStreamReader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +internal object YoutubeTakeoutCsvPortabilityReader { + private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" + + fun readManifests(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + csvEntries(entries).filter(::isPlaylistManifest).forEach { entry -> + sink.markCategory(PortabilityCategory.PLAYLISTS) + forEachRow(zip, entry) { header, row -> + val playlist = YoutubeTakeoutRowParser.parsePlaylist(header, row) ?: return@forEachRow + indexPlaylist(playlist, sink) + if (!isSystemPlaylist(playlist)) { + sink.write(PortabilityPlaylist(playlist.id, playlist.name, playlist.description, playlist.createdAt)) + } + } + } + } + + fun readContent(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + val csv = csvEntries(entries).filterNot(::isPlaylistManifest).toList() + csv.filter(::isSubscriptionFile).forEach { readSubscriptions(zip, it, sink) } + csv.filter(::isPlaylistContent).forEach { readPlaylistItems(zip, it, sink) } + } + + private fun readSubscriptions(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + forEachRow(zip, entry) { header, row -> + val item = YoutubeTakeoutRowParser.parseSubscription(header, row) + if (item == null) sink.invalid(PortabilityCategory.SUBSCRIPTIONS, "subscription") + else sink.write(item.toPortability()) + } + } + + private fun readPlaylistItems(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { + var inferredKey = playlistKeyFromPath(entry.name) + var rowPosition = 0 + forEachRow(zip, entry) { originalHeader, row -> + val hasKey = originalHeader.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + originalHeader.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + val header = if (hasKey || inferredKey == null) originalHeader else listOf("playlist source key") + originalHeader + val values = if (header === originalHeader) row else listOf(requireNotNull(inferredKey)) + row + if (YoutubeTakeoutRowParser.isUnavailablePlaylistItem(header, values)) return@forEachRow + val parsed = YoutubeTakeoutRowParser.parsePlaylistItem(header, values) + if (parsed == null) { + sink.invalid(PortabilityCategory.PLAYLISTS, "playlist item") + return@forEachRow + } + inferredKey = parsed.first + val item = parsed.second.copy(position = parsed.second.position.takeIf { it > 0 } ?: rowPosition) + writePlaylistVideo(parsed.first, item, sink) + rowPosition += 1 + } + } + + private fun writePlaylistVideo(key: String, item: dev.typetype.server.models.PlaylistVideoItem, sink: PortabilityRecordSink) { + when (val resolved = YoutubeTakeoutSystemPlaylist.canonicalKey(key) ?: sink.lookup(PLAYLIST_LOOKUP, key)) { + YoutubeTakeoutSystemPlaylist.WATCH_LATER -> sink.write( + PortabilityWatchLater(item.toPortabilityVideo(), item.addedAt), + ) + YoutubeTakeoutSystemPlaylist.LIKED_VIDEOS -> sink.write( + PortabilityFavorite(item.toPortabilityVideo(), item.addedAt), + ) + null -> { + val sourceId = key.trim() + sink.write(PortabilityPlaylist(sourceId, sourceId)) + sink.write(PortabilityPlaylistVideo(sourceId, item.position, item.toPortabilityVideo(), item.addedAt)) + sink.issue(PortabilityIssue(PortabilityCategory.PLAYLISTS, "playlist_manifest_missing", "A playlist was reconstructed from its item file")) + } + else -> sink.write(PortabilityPlaylistVideo(resolved, item.position, item.toPortabilityVideo(), item.addedAt)) + } + } + + private fun indexPlaylist(item: PlaylistItem, sink: PortabilityRecordSink) { + val resolved = YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) ?: YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) + ?: item.id.ifBlank { item.name } + if (item.id.isNotBlank()) sink.putLookup(PLAYLIST_LOOKUP, item.id, resolved) + playlistAliases(item.name).forEach { sink.putLookup(PLAYLIST_LOOKUP, it, resolved) } + } + + private fun forEachRow(zip: ZipFile, entry: ZipEntry, block: (List, List) -> Unit) { + zip.getInputStream(entry).use { input -> + val reader = BufferedReader(InputStreamReader(input, Charsets.UTF_8)) + var header = emptyList() + YoutubeTakeoutCsvReader.forEach(reader, { header = it }) { row -> block(header, row) } + } + } + + private fun csvEntries(entries: List) = entries.asSequence().filter { + it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() + } + + private fun isPlaylistManifest(entry: ZipEntry): Boolean = fileStem(entry) in setOf("playlists", "oynatma listeleri") + + private fun isSubscriptionFile(entry: ZipEntry): Boolean = fileStem(entry) in SUBSCRIPTION_NAMES + + private fun isPlaylistContent(entry: ZipEntry): Boolean { + val normalized = YoutubeTakeoutSchemaHints.normalize(entry.name) + return "playlist" in normalized || "oynatma list" in normalized + } + + private fun playlistKeyFromPath(path: String): String? = path.substringAfterLast('/').substringBeforeLast('.') + .takeUnless { YoutubeTakeoutSchemaHints.normalize(it) == "playlist items" } + ?.let { YoutubeTakeoutSystemPlaylist.canonicalKey(it) ?: it } + + private fun fileStem(entry: ZipEntry) = YoutubeTakeoutSchemaHints.normalize( + entry.name.substringAfterLast('/').substringBeforeLast('.'), + ) + + private fun isSystemPlaylist(item: PlaylistItem) = + YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null + + private fun playlistAliases(name: String): Set = setOf( + name, + "Videos from $name", + "Videos de $name", + "Vidéos de $name", + "Videos da playlist $name", + "$name videos", + ) + + private fun PortabilityRecordSink.invalid(category: PortabilityCategory, kind: String) = + issue(PortabilityIssue(category, "invalid_takeout_row", "An invalid YouTube Takeout $kind row was skipped")) + + private val SUBSCRIPTION_NAMES = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") +} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt new file mode 100644 index 00000000..e48116cf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +import dev.typetype.server.services.YoutubeTakeoutActivitySignalService +import dev.typetype.server.services.YoutubeTakeoutHistoryParser +import java.io.Reader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +internal object YoutubeTakeoutHtmlPortabilityReader { + fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> + zip.getInputStream(entry).bufferedReader().use { reader -> + readWindows(reader) { html -> writeWindow(html, sink) } + } + } + } + + private fun writeWindow(html: String, sink: PortabilityRecordSink) { + YoutubeTakeoutHistoryParser.parse(html).forEach { sink.write(it.toPortability()) } + val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) + subscriptions.forEach { sink.write(it.toPortability()) } + favorites.forEach { sink.write(it.toPortability()) } + } + + private fun readWindows(reader: Reader, block: (String) -> Unit) { + val buffer = CharArray(READ_CHARS) + val window = StringBuilder(WINDOW_CHARS + READ_CHARS) + while (true) { + val read = reader.read(buffer) + if (read < 0) break + window.append(buffer, 0, read) + if (window.length >= WINDOW_CHARS) { + block(window.toString()) + window.delete(0, window.length - OVERLAP_CHARS) + } + } + if (window.isNotEmpty()) block(window.toString()) + } + + private fun isYoutubeHtml(entry: ZipEntry): Boolean = + entry.name.endsWith(".html", ignoreCase = true) && "youtube" in entry.name.lowercase() + + private const val READ_CHARS = 32 * 1024 + private const val WINDOW_CHARS = 512 * 1024 + private const val OVERLAP_CHARS = 128 * 1024 +} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt new file mode 100644 index 00000000..b335dc2b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.portability + +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.HistoryItem +import dev.typetype.server.models.PlaylistVideoItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.YoutubeTypeTypeMapper + +internal fun SubscriptionItem.toPortability() = PortabilitySubscription( + channelUrl = channelUrl, + name = name, + avatarUrl = avatarUrl, + subscribedAt = subscribedAt, +) + +internal fun HistoryItem.toPortability() = PortabilityHistory( + video = PortabilityVideo( + url = url, + title = title, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + ), + watchedAt = watchedAt, + positionSeconds = progress, +) + +internal fun FavoriteItem.toPortability() = PortabilityFavorite( + video = PortabilityVideo( + url = videoUrl, + title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(videoUrl) }, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(videoUrl) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + viewCount = viewCount, + publishedAt = publishedAt, + ), + favoritedAt = favoritedAt, +) + +internal fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( + url = url, + title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(url) }, + thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, + durationSeconds = duration, + channelName = channelName, + channelUrl = channelUrl, + channelAvatarUrl = channelAvatar, + viewCount = viewCount, + publishedAt = publishedAt, +) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt new file mode 100644 index 00000000..50e3cbfc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.portability + +import java.io.OutputStream +import java.util.zip.ZipFile + +class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + format = PortabilityFormat.YOUTUBE_TAKEOUT, + adapterVersion = 1, + capabilities = TAKEOUT_CATEGORIES.mapTo(linkedSetOf()) { category -> + PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) + }, + defaultExtension = "zip", + contentType = "application/zip", + ) + + override fun detect(input: PortabilityInput): PortabilityDetection? { + val archive = input.archive ?: return null + val names = archive.names.map(String::lowercase) + val youtubeFiles = names.count { name -> + "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html")) + } + if (youtubeFiles == 0) return null + val hasTakeoutRoot = names.any { it.startsWith("takeout/") } + return PortabilityDetection( + PortabilityFormat.YOUTUBE_TAKEOUT, + null, + if (hasTakeoutRoot) 99 else 90, + "YouTube Takeout CSV or activity files", + ) + } + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + requireNotNull(detect(input)) { "Unsupported YouTube Takeout archive" } + ZipFile(input.path.toFile()).use { zip -> + val entries = zip.entries().asSequence().filterNot { it.isDirectory }.toList() + YoutubeTakeoutCsvPortabilityReader.readManifests(zip, entries, sink) + YoutubeTakeoutHtmlPortabilityReader.read(zip, entries, sink) + YoutubeTakeoutCsvPortabilityReader.readContent(zip, entries, sink) + } + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ): Unit = error("YouTube Takeout export is not available") +} + +private val TAKEOUT_CATEGORIES = setOf( + PortabilityCategory.SUBSCRIPTIONS, + PortabilityCategory.HISTORY, + PortabilityCategory.PLAYLISTS, + PortabilityCategory.WATCH_LATER, + PortabilityCategory.FAVORITES, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt index e3718d67..ffd33a88 100644 --- a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt +++ b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt @@ -10,27 +10,38 @@ import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond import io.ktor.server.response.respondOutputStream import okhttp3.Response +import java.net.URI suspend fun forwardDownloaderArtifactRequest( call: ApplicationCall, gateway: DownloaderGatewayService, - response: DownloaderGatewayResponse, + method: String, + path: String, + query: String?, requestHeaders: Map, forceDownload: Boolean, ) { - val location = artifactHeader(response, HttpHeaders.Location) - if (location == null) { - call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) - return - } - - val upstream = runCatching { gateway.openFetchAbsolute(location, requestHeaders) } + val upstream = runCatching { gateway.openForward(method, path, query, requestHeaders, null) } .getOrElse { call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) return } + val artifact = if (shouldProxyArtifact(upstream)) { + val location = upstream.header(HttpHeaders.Location) + upstream.close() + if (location == null) { + call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) + return + } + runCatching { gateway.openFetchAbsolute(location, method, requestHeaders) } + .getOrElse { + call.respond(HttpStatusCode.BadGateway, ErrorResponse("artifact unavailable")) + return + } + } else { + upstream + } - val artifact = upstream val headers = artifactHeaders(artifact) headers.forEach { (name, value) -> if (shouldForwardArtifactResponseHeader(name, forceDownload)) { @@ -44,8 +55,10 @@ suspend fun forwardDownloaderArtifactRequest( try { call.respondOutputStream(contentType = contentType, status = status) { artifact.use { response -> - response.body.byteStream().use { input -> - input.copyTo(this, DEFAULT_BUFFER_SIZE) + if (method != "HEAD") { + response.body.byteStream().use { input -> + input.copyTo(this, DEFAULT_BUFFER_SIZE) + } } } } @@ -55,9 +68,6 @@ suspend fun forwardDownloaderArtifactRequest( } } -private fun artifactHeader(response: DownloaderGatewayResponse, name: String): String? = - response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second - private fun artifactHeaders(response: Response): List> = response.headers.names().flatMap { name -> response.headers(name).map { name to it } } @@ -69,11 +79,25 @@ private fun artifactResponse(response: Response, headers: List if (shouldForwardGatewayResponseHeader(name, forceDownload)) { call.response.headers.append(name, value, safeOnly = false) @@ -97,21 +96,3 @@ private fun isSseRequest(path: String, headers: Map): Boolean { val accept = headers.entries.firstOrNull { it.key.equals("Accept", ignoreCase = true) }?.value.orEmpty() return accept.contains("text/event-stream", ignoreCase = true) } - -private fun shouldProxyArtifact(path: String, response: dev.typetype.server.services.DownloaderGatewayResponse): Boolean { - if (!path.endsWith("/artifact")) return false - if (response.status != 302 && response.status != 307) return false - val location = headerValue(response, "Location") ?: return false - val markedInternal = headerValue(response, INTERNAL_ARTIFACT_PROXY_HEADER) == "1" - return markedInternal || isLegacyInternalHost(location) -} - -private fun headerValue(response: dev.typetype.server.services.DownloaderGatewayResponse, name: String): String? = - response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second - -private fun isLegacyInternalHost(location: String): Boolean { - val host = runCatching { URI(location).host }.getOrNull() ?: return false - return host.equals("garage", ignoreCase = true) -} - -private const val INTERNAL_ARTIFACT_PROXY_HEADER = "X-TypeType-Artifact-Proxy" diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt new file mode 100644 index 00000000..a93d672f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.routes + +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityImportRequest +import dev.typetype.server.services.AuthService +import io.ktor.http.ContentDisposition +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.call +import io.ktor.server.request.receive +import io.ktor.server.response.header +import io.ktor.server.response.respond +import io.ktor.server.response.respondFile +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +internal fun Route.portabilityJobRoutes(engine: PortabilityEngine, authService: AuthService) { + get("/portability/jobs/{id}") { withJobAccount(authService) { owner, id -> call.respond(engine.snapshot(owner, id)) } } + get("/portability/jobs/{id}/report") { + withJobAccount(authService) { owner, id -> call.respond(engine.report(owner, id)) } + } + post("/portability/jobs/{id}/apply") { + withJobAccount(authService) { owner, id -> + call.respond(HttpStatusCode.Accepted, engine.applyImport(owner, id, call.receive())) + } + } + post("/portability/jobs/{id}/cancel") { + withJobAccount(authService) { owner, id -> call.respond(engine.cancel(owner, id)) } + } + get("/portability/jobs/{id}/artifact") { + withJobAccount(authService) { owner, id -> + val artifact = engine.artifact(owner, id) + call.response.header( + HttpHeaders.ContentDisposition, + ContentDisposition.Attachment + .withParameter(ContentDisposition.Parameters.FileName, artifact.fileName.toString()) + .toString(), + ) + call.respondFile(artifact.toFile()) + } + } + delete("/portability/jobs/{id}") { + withJobAccount(authService) { owner, id -> + engine.delete(owner, id) + call.respond(HttpStatusCode.NoContent) + } + } +} + +private suspend fun io.ktor.server.routing.RoutingContext.withJobAccount( + authService: AuthService, + block: suspend (String, String) -> Unit, +) { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> + val id = call.parameters["id"] + if (id == null) { + call.respondPortabilityError(IllegalArgumentException("Missing portability job id")) + } else { + runCatching { block(owner, id) } + .onFailure { call.respondPortabilityError(it as? Exception ?: RuntimeException(it)) } + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt new file mode 100644 index 00000000..28ac1d61 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.portability.PortabilityContractException +import dev.typetype.server.portability.PortabilityFormat +import dev.typetype.server.portability.PortabilityJobNotFoundException +import dev.typetype.server.portability.PortabilityUploadTooLargeException +import dev.typetype.server.portability.portabilityErrorCode +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +internal fun parsePortabilityFormat(value: String?): PortabilityFormat? { + if (value == null) return null + return PortabilityFormat.entries.firstOrNull { it.wireName == value } + ?: throw IllegalArgumentException("Unsupported portability format") +} + +internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) { + val status = when (error) { + is PortabilityJobNotFoundException -> HttpStatusCode.NotFound + is PortabilityUploadTooLargeException -> HttpStatusCode.PayloadTooLarge + is IllegalStateException -> HttpStatusCode.Conflict + is IllegalArgumentException -> HttpStatusCode.BadRequest + else -> HttpStatusCode.InternalServerError + } + val message = if (error is PortabilityContractException || error is IllegalArgumentException) { + error.message ?: "Invalid portability request" + } else { + "Portability operation failed" + } + respond(status, ErrorResponse(message, portabilityErrorCode(error))) +} + +internal suspend inline fun ApplicationCall.withPortabilityAccount( + userId: String, + crossinline block: suspend (String) -> Unit, +) { + if (userId.startsWith("guest:")) { + respond(HttpStatusCode.Forbidden, ErrorResponse("Guest users do not have portable account data")) + return + } + block(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt new file mode 100644 index 00000000..ae05d946 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt @@ -0,0 +1,77 @@ +package dev.typetype.server.routes + +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityExportRequest +import dev.typetype.server.portability.PortabilityUploadWriter +import dev.typetype.server.requestId +import dev.typetype.server.services.AuthService +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.PartData +import io.ktor.server.application.call +import io.ktor.server.request.receive +import io.ktor.server.request.receiveMultipart +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import java.nio.file.Files + +fun Route.portabilityRoutes(engine: PortabilityEngine, authService: AuthService) { + get("/portability/formats") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { call.respond(engine.formats()) } + } + } + post("/portability/imports") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> call.uploadImport(engine, owner) } + } + } + post("/portability/exports") { + call.withJwtAuth(authService) { userId -> + call.withPortabilityAccount(userId = userId) { owner -> + runCatching { + val request = call.receive() + engine.startExport(owner, request.format, request.categories, call.requestId()) + } + .onSuccess { call.respond(HttpStatusCode.Accepted, it) } + .onFailure { call.respondPortabilityError(it.asException()) } + } + } + } + portabilityJobRoutes(engine, authService) +} + +private suspend fun io.ktor.server.application.ApplicationCall.uploadImport(engine: PortabilityEngine, owner: String) { + val tmp = Files.createTempFile("portability-upload-", ".tmp") + try { + val multipart = receiveMultipart(dev.typetype.server.portability.PortabilityLimits.MAX_UPLOAD_BYTES) + var filename: String? = null + var contentType: String? = null + var files = 0 + while (true) { + val part = multipart.readPart() ?: break + if (part is PartData.FileItem && part.name == "file") { + files += 1 + if (files == 1) { + filename = part.originalFileName ?: "backup" + contentType = part.contentType?.toString() + PortabilityUploadWriter.write(part.provider(), tmp) + } + } + part.release() + } + require(files == 1) { "Exactly one backup file is required" } + val hint = parsePortabilityFormat(request.queryParameters["format"]) + respond( + HttpStatusCode.Accepted, + engine.startImportPreview(owner, tmp, requireNotNull(filename), contentType, hint, requestId()), + ) + } catch (error: Exception) { + respondPortabilityError(error) + } finally { + Files.deleteIfExists(tmp) + } +} + +private fun Throwable.asException(): Exception = this as? Exception ?: RuntimeException(this) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt b/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt index b17ad40c..3998300f 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt @@ -5,6 +5,12 @@ import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond import io.ktor.server.response.respondBytes +import io.ktor.server.response.respondOutputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import java.io.EOFException +import java.io.InputStream +import java.io.OutputStream internal suspend fun ApplicationCall.respondSabrMediaBytes(mimeType: String, body: ByteArray): Unit { val total = body.size.toLong() @@ -26,3 +32,54 @@ internal suspend fun ApplicationCall.respondSabrMediaBytes(mimeType: String, bod null -> respondBytes(body, containerMime(mimeType), HttpStatusCode.OK) } } + +internal suspend fun ApplicationCall.respondSabrMediaStream( + mimeType: String, + total: Long, + openStream: () -> InputStream, + onOpened: () -> Unit, +): Unit { + response.headers.append(HttpHeaders.CacheControl, "no-store") + response.headers.append(HttpHeaders.AcceptRanges, "bytes") + when (val range = parseAudioOnlyByteRange(request.headers[HttpHeaders.Range], total)) { + is AudioOnlyByteRange.Satisfiable -> { + response.headers.append(HttpHeaders.ContentRange, "bytes ${range.first}-${range.last}/${range.total}") + stream(openStream, onOpened, range.first, range.last - range.first + 1L, HttpStatusCode.PartialContent, mimeType) + } + is AudioOnlyByteRange.Unsatisfiable -> { + response.headers.append(HttpHeaders.ContentRange, "bytes */${range.total}") + respond(HttpStatusCode.RequestedRangeNotSatisfiable) + } + null -> stream(openStream, onOpened, 0L, total, HttpStatusCode.OK, mimeType) + } +} + +private suspend fun ApplicationCall.stream( + openStream: () -> InputStream, + onOpened: () -> Unit, + offset: Long, + length: Long, + status: HttpStatusCode, + mimeType: String, +): Unit = respondOutputStream(containerMime(mimeType), status, length) { + openStream().use { input -> + runInterruptible(Dispatchers.IO) { + input.skipNBytes(offset) + onOpened() + copyExactly(input, length) + } + } +} + +private fun OutputStream.copyExactly(input: InputStream, length: Long) { + val buffer = ByteArray(COPY_BUFFER_SIZE) + var remaining = length + while (remaining > 0L) { + val read = input.read(buffer, 0, minOf(buffer.size.toLong(), remaining).toInt()) + if (read < 0) throw EOFException("SABR media segment ended before its declared length") + write(buffer, 0, read) + remaining -= read + } +} + +private const val COPY_BUFFER_SIZE = 64 * 1024 diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 0e544d4b..6326898a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -14,6 +14,7 @@ import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService +import dev.typetype.server.services.markServed import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -118,6 +119,11 @@ internal class SabrPlaybackHandler( private suspend fun respondSegment(call: ApplicationCall, result: SabrPlaybackSegmentResult): Unit = when (result) { is SabrPlaybackSegmentResult.Ready -> call.respondSabrMediaBytes(result.mimeType, result.bytes) + is SabrPlaybackSegmentResult.Stream -> call.respondSabrMediaStream( + result.mimeType, + result.segment.length.toLong(), + result.segment::openStream, + ) { result.holder.markServed(result.segment, result.generation) } is SabrPlaybackSegmentResult.Retry -> call.respond( HttpStatusCode.Accepted, result.holder.toRetryPlaybackResponse(result.status, RETRY_AFTER_MS), @@ -128,6 +134,7 @@ internal class SabrPlaybackHandler( } private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { + if (!access.profile.enabled) return true return when (val result = accessValidator.resolve(access.userId, videoId)) { is ExtractionResult.Success -> { val allowed = !access.profile.enabled || diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt index 82c8113a..58f28b67 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -12,7 +12,6 @@ import dev.typetype.server.services.livePlaybackSnapshot import dev.typetype.server.services.playbackContinuationSequence import dev.typetype.server.services.playbackSegmentStartMs import dev.typetype.server.services.resolvePlaybackStartMs -import dev.typetype.server.services.playbackStartSequence import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat @@ -29,11 +28,12 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi if (effectiveRequest.audioOnly) return buildAudioOnly(holder, effectiveRequest, live?.toResponse()) val video = buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, live?.active == true) val decodeStartMs = video.track.segments.firstOrNull()?.startMs ?: effectiveRequest.playerTimeMs + val audioStartMs = if (live?.active == true) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs val audio = buildTrack( holder, holder.audioFormat, effectiveRequest, - minOf(effectiveRequest.playerTimeMs, decodeStartMs), + audioStartMs, live?.active == true, ) val playbackStartMs = resolvedPlaybackStartMs( @@ -93,7 +93,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi sequenceOf(video, audio) .filter { it.blockedRequest != null } .minByOrNull { it.coveredEndMs } - private suspend fun buildTrack( holder: SabrSessionHolder, format: YoutubeSabrFormat, @@ -134,6 +133,17 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi } } if (segment == null) { + val progressive = if (activeLive) null else segments.appendProgressiveWindowSegment( + holder, format, seq, expectedStartMs, + ) + if (progressive != null) { + seq = progressive.nextSequence + coveredEndMs = progressive.coveredEndMs + if (coveredEndMs >= goalEndMs) break + blockedRequest = SabrSegmentRequest.media(format, seq) + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + break + } blockedBy = "${format.trackName()}:${format.itag}:$seq pending" blockedRequest = mediaRequest break @@ -169,12 +179,11 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi if (blockedBy == null && coveredEndMs < goalEndMs && !atEnd) { blockedBy = "${format.trackName()}:${format.itag}:$seq window capped" } - val mediaBasePath = SabrPlaybackPaths.mediaBasePath(holder.sessionToken) - val defaultInitUrl = "$mediaBasePath/${format.itag}/init?generation=${holder.activeGeneration()}" return TrackBuildResult( track = SabrPlaybackWindowTrack( mime = format.mimeType.orEmpty(), - initUrl = defaultInitUrl, + initUrl = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/init" + + "?generation=${holder.activeGeneration()}", segments = segments, ), blockedBy = blockedBy, @@ -183,7 +192,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi atEnd = atEnd, ) } - private fun CachedSabrSegment.toWindowSegment( holder: SabrSessionHolder, format: YoutubeSabrFormat, @@ -197,7 +205,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi durationMs = resolvedDurationMs, ) } - private data class TrackBuildResult( val track: SabrPlaybackWindowTrack, val blockedBy: String?, @@ -207,7 +214,6 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi ) { fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs } - private companion object { const val MAX_SEGMENTS_PER_TRACK = 12 } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt index b25629bf..52d4d2c9 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt @@ -30,7 +30,7 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi sabrSessionStore.startPump(holder) val window = buildWithTargetedPrefetch(holder, request) - if (window.isReady) { + if (holder.canServe(window)) { return call.respond(HttpStatusCode.OK, window.response) } call.respond(HttpStatusCode.Accepted, holder.preparingResponse(request, window)) @@ -68,8 +68,8 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi holder.applyClientPreferences() sabrSessionStore.startPump(holder) val window = buildWithTargetedPrefetch(holder, request) - val status = if (window.isReady) HttpStatusCode.OK else HttpStatusCode.Accepted - call.respond(status, holder.prefetchResponse(request, window)) + val ready = holder.canServe(window) + call.respond(if (ready) HttpStatusCode.OK else HttpStatusCode.Accepted, holder.prefetchResponse(request, window, ready)) } suspend fun segments(call: ApplicationCall, sessionId: String) { @@ -80,7 +80,7 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) holder.applyClientPreferences() val window = windowBuilder.build(holder, request) - if (window.isReady) return call.respond(HttpStatusCode.OK, window.response) + if (holder.canServe(window)) return call.respond(HttpStatusCode.OK, window.response) call.respond(HttpStatusCode.Accepted, holder.preparingResponse(request, window)) } @@ -95,6 +95,9 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi return window } + private fun SabrSessionHolder.canServe(window: SabrPlaybackWindowBuildResult): Boolean = + window.isReady && terminalFailure() == null && networkFailure() == null + private suspend fun SabrSessionHolder.preparingResponse( request: SabrPlaybackWindowRequest, window: SabrPlaybackWindowBuildResult, @@ -122,15 +125,16 @@ internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessi private suspend fun SabrSessionHolder.prefetchResponse( request: SabrPlaybackWindowRequest, window: SabrPlaybackWindowBuildResult, + ready: Boolean, ): SabrPlaybackPrefetchResponse = SabrPlaybackPrefetchResponse( sessionId = sessionToken, generation = activeGeneration(), - ready = window.isReady, - retryAfterMs = if (window.isReady) null else liveRetryAfterMs(window.blockedRequests), + ready = ready, + retryAfterMs = if (ready) null else liveRetryAfterMs(window.blockedRequests), status = playbackState().name.lowercase(), segmentsUrl = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/segments", stateUrl = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/state", - blockedBy = if (window.isReady) null else SabrPlaybackDiagnostics.blocker(this) ?: window.blockedBy ?: "window pending", + blockedBy = if (ready) null else SabrPlaybackDiagnostics.blocker(this) ?: window.blockedBy ?: "window pending", playerTimeMs = request.playerTimeMs.coerceAtLeast(0L), readerHeadMs = readerHeadMs(), readerTailMs = readerTailMs(), diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt new file mode 100644 index 00000000..cc14f9e0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.findCachedMediaAt +import dev.typetype.server.services.playbackSegmentDurationMs +import dev.typetype.server.services.playbackSegmentStartMs +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +internal data class SabrProgressiveWindowSegment( + val sequence: Int, + val response: SabrPlaybackWindowSegment, +) + +internal data class SabrProgressiveWindowAppend(val nextSequence: Int, val coveredEndMs: Long) + +internal fun MutableList.appendProgressiveWindowSegment( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + predictedSequence: Int, + targetMs: Long, +): SabrProgressiveWindowAppend? { + val segment = holder.progressiveWindowSegment(format, predictedSequence, targetMs) ?: return null + if (segment.sequence != predictedSequence) holder.session.streamState.jumpBufferedTo(format, segment.sequence) + add(segment.response) + return SabrProgressiveWindowAppend( + segment.sequence + 1, + segment.response.startMs + segment.response.durationMs, + ) +} + +internal fun SabrSessionHolder.progressiveWindowSegment( + format: YoutubeSabrFormat, + predictedSequence: Int, + targetMs: Long, +): SabrProgressiveWindowSegment? { + val request = SabrSegmentRequest.media(format, predictedSequence) + val exact = session.getReadableSegment(request)?.takeIf { it.covers(targetMs, this, format) } + val actual = exact ?: session.findCachedMediaAt(format, targetMs, predictedSequence) + actual?.let(::observeMediaSegment) + val sequence = actual?.header?.sequenceNumber ?: predictedSequence + val startMs = actual?.header?.startMs?.takeIf { it >= 0L } + ?: playbackSegmentStartMs(format, sequence) + val durationMs = actual?.header?.durationMs?.takeIf { it > 0L } + ?: playbackSegmentDurationMs(format, sequence) + if (startMs < 0L || durationMs <= 0L) return null + return SabrProgressiveWindowSegment( + sequence, + SabrPlaybackWindowSegment( + url = "${SabrPlaybackPaths.mediaBasePath(sessionToken)}/${format.itag}/segment/$sequence?generation=${activeGeneration()}", + startMs = startMs, + durationMs = durationMs, + ), + ) +} + +private fun SabrMediaSegment.covers( + targetMs: Long, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, +): Boolean { + val startMs = header.startMs.takeIf { it >= 0L } + ?: holder.playbackSegmentStartMs(format, header.sequenceNumber) + val durationMs = header.durationMs.takeIf { it > 0L } + ?: holder.playbackSegmentDurationMs(format, header.sequenceNumber) + return targetMs >= startMs - TIMING_TOLERANCE_MS && targetMs < startMs + durationMs +} + +private const val TIMING_TOLERANCE_MS = 2L diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt new file mode 100644 index 00000000..022aeaa6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionGroupMembershipRequest +import dev.typetype.server.services.SubscriptionGroupsService +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.contentType +import io.ktor.server.request.receiveChannel +import io.ktor.server.response.respond +import io.ktor.utils.io.ByteReadChannel +import io.ktor.utils.io.jvm.javaio.toInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import java.io.ByteArrayOutputStream + +private const val MAX_MEMBERSHIP_REQUEST_BODY_BYTES = 1024 * 1024 +private const val MAX_MEMBERSHIP_CHANNEL_URL_LENGTH = 2048 +private val membershipRequestJson = Json { ignoreUnknownKeys = true } + +internal sealed interface MembershipChannels { + data class Single(val channelUrl: String) : MembershipChannels + data class Batch(val channelUrls: List) : MembershipChannels +} + +internal suspend fun ApplicationCall.receiveMembershipChannels(body: ByteArray): MembershipChannels? { + val request = if (request.contentType().match(ContentType.Application.Json)) { + try { + membershipRequestJson.decodeFromString(body.decodeToString()) + } catch (_: SerializationException) { + null + } + } else { + null + } + val channelUrl = request?.channelUrl?.takeIf(String::isValidMembershipChannelUrl) + val channelUrls = request?.channelUrls + val parsed = when { + channelUrl != null && channelUrls == null -> MembershipChannels.Single(channelUrl) + request?.channelUrl == null && channelUrls != null && + channelUrls.size in 1..SubscriptionGroupsService.MAX_MEMBERSHIP_CHANNELS && + channelUrls.all(String::isValidMembershipChannelUrl) -> MembershipChannels.Batch(channelUrls) + else -> null + } + if (parsed == null) respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + return parsed +} + +internal suspend fun ApplicationCall.receiveMembershipBody(): ByteArray? { + val contentLength = request.headers[HttpHeaders.ContentLength]?.toLongOrNull() + if (contentLength != null && contentLength > MAX_MEMBERSHIP_REQUEST_BODY_BYTES) { + respondMembershipBodyTooLarge() + return null + } + val body = receiveChannel().readUpTo(MAX_MEMBERSHIP_REQUEST_BODY_BYTES) + if (body == null) respondMembershipBodyTooLarge() + return body +} + +private suspend fun ByteReadChannel.readUpTo(maxBytes: Int): ByteArray? = withContext(Dispatchers.IO) { + toInputStream().use { input -> + ByteArrayOutputStream().use { output -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read <= 0) break + if (output.size() + read > maxBytes) return@withContext null + output.write(buffer, 0, read) + } + output.toByteArray() + } + } +} + +internal fun String.isValidMembershipChannelUrl(): Boolean = + isNotBlank() && length <= MAX_MEMBERSHIP_CHANNEL_URL_LENGTH + +private suspend fun ApplicationCall.respondMembershipBodyTooLarge() = respond( + HttpStatusCode.PayloadTooLarge, + ErrorResponse("Request body exceeds 1 MiB", "request_body_too_large"), +) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt index c80acb3d..2109f4b7 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -1,7 +1,6 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.models.SubscriptionGroupMembershipRequest import dev.typetype.server.models.SubscriptionGroupRequest import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionGroupMembershipResult @@ -45,21 +44,44 @@ fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, aut put("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - val request = runCatching { call.receive() }.getOrElse { - return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + val body = call.receiveMembershipBody() ?: return@withJwtAuth + when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { + is MembershipChannels.Single -> call.respondMembership( + groupsService.addSubscription(userId, groupId, request.channelUrl), + ) + is MembershipChannels.Batch -> call.respondMembership( + groupsService.addSubscriptions(userId, groupId, request.channelUrls), + ) } - if (request.channelUrl.isBlank()) { - return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("channelUrl must not be blank")) - } - call.respondMembership(groupsService.addSubscription(userId, groupId, request.channelUrl)) } } delete("/subscriptions/groups/{groupId}/channels") { call.withJwtAuth(authService) { userId -> val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() - val channelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) - ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) - call.respondMembership(groupsService.removeSubscription(userId, groupId, channelUrl)) + val queryChannelUrl = call.request.queryParameters["url"] + if (queryChannelUrl != null && !queryChannelUrl.isValidMembershipChannelUrl()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid channel URL")) + } + val body = call.receiveMembershipBody() ?: return@withJwtAuth + if (queryChannelUrl != null) { + if (body.isNotEmpty()) { + return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Specify either url or a request body, not both"), + ) + } + return@withJwtAuth call.respondMembership( + groupsService.removeSubscription(userId, groupId, queryChannelUrl), + ) + } + when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { + is MembershipChannels.Single -> call.respondMembership( + groupsService.removeSubscription(userId, groupId, request.channelUrl), + ) + is MembershipChannels.Batch -> call.respondMembership( + groupsService.removeSubscriptions(userId, groupId, request.channelUrls), + ) + } } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index becd5d9b..33aac871 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -27,6 +27,11 @@ fun Route.subscriptionsRoutes( warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), ) { + get("/subscriptions/group-memberships") { + call.withJwtAuth(authService) { userId -> + call.respond(subscriptionsService.getAllWithGroupMemberships(userId)) + } + } get("/subscriptions") { call.withJwtAuth(authService) { userId -> val parsed = call.parseSubscriptionSelection() diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 1ff721c8..9da6de79 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -6,6 +6,7 @@ import dev.typetype.server.services.AvatarService import dev.typetype.server.services.BugReportService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService +import dev.typetype.server.portability.PortabilityEngine import io.ktor.server.routing.Route internal fun Route.userDataRoutes( @@ -15,6 +16,7 @@ internal fun Route.userDataRoutes( avatarService: AvatarService, bugReportService: BugReportService, restoreService: PipePipeBackupImporterService, + portabilityEngine: PortabilityEngine, ) { historyRoutes(svc.historyService, authService, svc.settingsService) subscriptionGroupsRoutes(svc.subscriptionGroupsService, authService) @@ -50,6 +52,7 @@ internal fun Route.userDataRoutes( bugReportRoutes(bugReportService, authService) restoreRoutes(restoreService, authService) typeTypeBackupRoutes(svc.typeTypeBackupService, authService) + portabilityRoutes(portabilityEngine, authService) homeRecommendationRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) homeRecommendationShortsRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt index 063e63ce..1772414c 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -1,12 +1,18 @@ package dev.typetype.server.services import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withTimeout import java.time.Duration import java.util.concurrent.ConcurrentHashMap internal class AuthenticatedSabrInfoCache( ttl: Duration = Duration.ofMinutes(5), maxEntries: Int = 256, + private val timeoutMs: Long = AuthenticatedSabrPolicy.INFO_TIMEOUT_MS, ) { private val items = BoundedExpiringCache( maxEntries = maxEntries, @@ -23,9 +29,9 @@ internal class AuthenticatedSabrInfoCache( items.get(key)?.let { return AuthenticatedSabrInfoResult.Ready(it) } val pending = CompletableDeferred() val existing = inFlight.putIfAbsent(key, pending) - if (existing != null) return existing.await() + if (existing != null) return awaitExisting(credentials, videoId, loader, existing) return try { - val result = loader() + val result = withTimeout(timeoutMs) { loader() } if (result is AuthenticatedSabrInfoResult.Ready) items.put(key, result.prepared) pending.complete(result) result @@ -37,6 +43,20 @@ internal class AuthenticatedSabrInfoCache( } } + private suspend fun awaitExisting( + credentials: YoutubeSessionCredentials, + videoId: String, + loader: suspend () -> AuthenticatedSabrInfoResult, + pending: CompletableDeferred, + ): AuthenticatedSabrInfoResult = try { + withTimeout(timeoutMs) { pending.await() } + } catch (error: TimeoutCancellationException) { + throw error + } catch (error: CancellationException) { + if (!currentCoroutineContext().isActive) throw error + getOrLoad(credentials, videoId, loader) + } + private data class Key( val userId: String, val credentialFingerprint: String, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index b31ddb9f..66ecdecc 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -2,7 +2,9 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withContext +import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken @@ -22,7 +24,12 @@ internal class AuthenticatedSabrInfoService( if (userId == null || userId.startsWith("guest:")) return AuthenticatedSabrInfoResult.NotConnected val credentials = youtubeSessionService.connectedCredentials(userId) ?: return AuthenticatedSabrInfoResult.NotConnected - return cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + return try { + cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + } catch (error: TimeoutCancellationException) { + logger.warn("authenticated_sabr_probe event=timeout videoId={}", videoId) + AuthenticatedSabrInfoResult.TimedOut + } } private suspend fun fetchUncached( @@ -32,10 +39,10 @@ internal class AuthenticatedSabrInfoService( return try { val prepared = YoutubeSessionTokenScope.withCredentials(credentials) { withContext(Dispatchers.IO) { - val sessionBinding = visitorDataFetcher() - val token = tokenClient.fetchSession(videoId, sessionBinding) + val sessionBinding = runInterruptible(Dispatchers.IO) { visitorDataFetcher() } + val token = runInterruptible(Dispatchers.IO) { tokenClient.fetchSession(videoId, sessionBinding) } ?: error("Token service did not return authenticated SABR tokens") - val info = probe.fetch(videoId, token.youtubeSessionPoToken()) + val info = runInterruptible(Dispatchers.IO) { probe.fetch(videoId, token.youtubeSessionPoToken()) } SabrPreparedInfo( info = info, initialToken = token, @@ -67,6 +74,7 @@ internal class AuthenticatedSabrInfoService( internal sealed interface AuthenticatedSabrInfoResult { data object NotConnected : AuthenticatedSabrInfoResult data object Failed : AuthenticatedSabrInfoResult + data object TimedOut : AuthenticatedSabrInfoResult data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt new file mode 100644 index 00000000..a0a67f70 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt @@ -0,0 +1,7 @@ +package dev.typetype.server.services + +internal object AuthenticatedSabrPolicy { + const val INFO_TIMEOUT_MS = 15_000L + const val STREAM_TIMEOUT_MS = 20_000L + const val TIMEOUT_CODE = "authenticated_sabr_timeout" +} diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt index b25c9f0f..e19625ff 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt +++ b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt @@ -29,6 +29,17 @@ internal object BugReportValidation { if (context.apiErrors.any { it.endpoint.isBlank() || it.timestamp <= 0 || it.status <= 0 }) { return "Invalid api error entry" } + val dimensions = listOf( + context.viewportWidth, + context.viewportHeight, + context.screenWidth, + context.screenHeight, + ).filterNotNull() + if (dimensions.any { it !in 1..100_000 }) return "Invalid display dimensions" + if (context.devicePixelRatio != null && context.devicePixelRatio !in 0.1..100.0) { + return "Invalid device pixel ratio" + } + if (context.timezone != null && context.timezone.length > 128) return "Invalid timezone" return null } } diff --git a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt index c6a96745..56ef7de0 100644 --- a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt +++ b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt @@ -64,8 +64,8 @@ class DownloaderGatewayService( return httpClient.newCall(requestBuilder.build()).execute() } - fun openFetchAbsolute(url: String, headers: Map): Response { - val requestBuilder = Request.Builder().url(url).method("GET", null) + fun openFetchAbsolute(url: String, method: String, headers: Map): Response { + val requestBuilder = Request.Builder().url(url).method(method, null) headerValue(headers, "Range")?.takeIf { it.isNotBlank() }?.let { requestBuilder.addHeader("Range", it) } return client.newCall(requestBuilder.build()).execute() } diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index 3225e2d6..ab42ccb1 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -48,6 +48,7 @@ class InstanceService( youtubeRemoteLoginEnabled = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginReady = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginUnavailableReason = youtubeRemoteLoginStatus.unavailableReason, + parentalControlsEnabled = settings.accessMode == ACCESS_MODE_ALLOW_LIST, rss = RssInstanceCapability( enabled = settings.rssEnabled && settings.rssPublicBaseUrl != null, maxFeedsPerUser = settings.rssMaxFeedsPerUser, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt index a1998fb9..771aed8c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -2,24 +2,21 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope internal class SabrBootstrapStreamService( private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, ) : StreamService { - override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { + override suspend fun getStreamInfo(url: String): ExtractionResult { val videoId = youtubeVideoId(url) - ?: return@coroutineScope ExtractionResult.BadRequest("Invalid YouTube URL") - val prepared = async { sessionStore.fetchInfo(videoId, cachedFirst = true) } - val session = async { tokenSessionClient.fetchPlaybackSession(videoId) } - val metadata = session.await() - ?: return@coroutineScope ExtractionResult.Failure("SABR bootstrap metadata unavailable") - if (prepared.await() == null) { - return@coroutineScope ExtractionResult.Failure("SABR playback formats unavailable") - } - ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) + ?: return ExtractionResult.BadRequest("Invalid YouTube URL") + val metadata = tokenSessionClient.fetchPlaybackSession(videoId) + ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") + val prepared = metadata.preparedSabrInfo() + ?: sessionStore.fetchInfo(videoId, cachedFirst = true) + ?: return ExtractionResult.Failure("SABR playback formats unavailable") + sessionStore.rememberPreparedInfo(videoId, prepared) + return ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index e0186344..3e733749 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -26,6 +26,7 @@ internal object SabrDemandAttemptFinisher { if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false holder.clearSegmentDemands() val message = "SABR demand stalled for ${demand.request.summary()}" + SabrPumpLogger.expired(holder, demand.request, recoverable) holder.failTerminal(if (recoverable) sabrRecoverableFailureMessage(message) else message) true } @@ -40,6 +41,7 @@ internal object SabrDemandAttemptFinisher { if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false val current = holder.nextSegmentDemand() ?: return@synchronized false if (!current.matches(request) || holder.segmentDemandIdentity(current) != identity) return@synchronized false + SabrPumpLogger.expired(holder, request, recoverable) fail(holder, request, identity, recoverable) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index aba331af..2a0e3874 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -78,5 +78,5 @@ internal class SabrDemandWatchdog( } private fun nextCheckDelayMs(backoffRemainingMs: Long, futureLiveRequest: Boolean): Long = - maxOf(intervalMs, backoffRemainingMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) + maxOf(intervalMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index 6e5e5e74..e71c0bb5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -59,9 +59,12 @@ internal class SabrInfoFetcher( suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit { repository.rememberInitialization(videoId, info) - fetchInfoOnce(videoId, startTimeMs = 0L) - ?.takeIf { it.hasAudioAndVideoFormats() } - ?.let { repository.putPrepared(videoId, startTimeMs = 0L, it) } + fetchInfo(videoId, startTimeMs = 0L, cachedFirst = true) + } + + suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit { + repository.rememberInitialization(videoId, prepared.info) + repository.putPrepared(videoId, startTimeMs = 0L, prepared) } suspend fun invalidatePlayback(videoId: String): Unit = repository.invalidatePlayback(videoId) @@ -87,13 +90,7 @@ internal class SabrInfoFetcher( ): SabrPreparedInfo? = withTimeoutOrNull(SabrSessionStoreDefaults.INFO_TIMEOUT_MS) { val tokenSession = sessionClient?.fetchPlaybackSession(videoId, isolatedPlayback) - tokenSession?.token - ?.takeIf { it.visitorData == tokenSession.info.visitorData } - ?.let { sessionToken -> - SabrPreparedInfo(tokenSession.info, sessionToken, tokenSession.isLive, tokenSession.isLiveContent) - .takeIf { it.hasAudioAndVideoFormats() } - ?.let { return@withTimeoutOrNull it } - } + tokenSession?.preparedSabrInfo()?.let { return@withTimeoutOrNull it } val token = tokenClient.fetch(videoId) ?: return@withTimeoutOrNull null.also { logger.warn( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt index 4486e836..0df44214 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -10,7 +10,7 @@ internal class SabrPlaybackInfoResolver( startTimeMs: Long, ): SabrPreparedInfo? = when (val authenticated = authenticatedInfoService?.fetch(userId, videoId)) { is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared - AuthenticatedSabrInfoResult.Failed -> null + AuthenticatedSabrInfoResult.Failed, AuthenticatedSabrInfoResult.TimedOut -> null AuthenticatedSabrInfoResult.NotConnected, null -> sessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) } @@ -22,6 +22,7 @@ internal class SabrPlaybackInfoResolver( return when (val authenticated = authenticatedInfoService?.fetch(holder.key.userId, holder.key.videoId)) { is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared AuthenticatedSabrInfoResult.Failed, + AuthenticatedSabrInfoResult.TimedOut, AuthenticatedSabrInfoResult.NotConnected, null, -> null diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt index b343d061..10064135 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt @@ -1,5 +1,7 @@ package dev.typetype.server.services +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withTimeoutOrNull internal data class SabrPlaybackInitializationPreloadResult( @@ -23,10 +25,12 @@ internal object SabrPlaybackInitializationPreloader { audioOnly: Boolean, timeoutMs: Long, ): SabrPlaybackInitializationPreloadResult = withTimeoutOrNull(timeoutMs) { - val video = holder.videoFormat - .takeUnless { audioOnly } - ?.let { sessionStore.fetchInitializationData(holder, it) } - val audio = sessionStore.fetchInitializationData(holder, holder.audioFormat) - SabrPlaybackInitializationPreloadResult(video, audio) + coroutineScope { + val video = holder.videoFormat + .takeUnless { audioOnly } + ?.let { format -> async { sessionStore.fetchInitializationData(holder, format) } } + val audio = async { sessionStore.fetchInitializationData(holder, holder.audioFormat) } + SabrPlaybackInitializationPreloadResult(video?.await(), audio.await()) + } } ?: SabrPlaybackInitializationPreloadResult(null, null) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt new file mode 100644 index 00000000..4ba20e8e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt @@ -0,0 +1,110 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withTimeoutOrNull +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest + +internal class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { + suspend fun fetch( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult = if (holder.livePlaybackSnapshot()?.active == true) { + fetchLive(holder, request, timeoutMs, generation) + } else { + fetchProgressive(holder, request, timeoutMs, generation) + } + + private suspend fun fetchProgressive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult { + holder.session.getReadableSegment(request)?.let { + return stream(holder, request, it, generation) + } + sessionStore.requestSegmentDemand(holder, request, generation) + val segment = runInterruptible(Dispatchers.IO) { + holder.session.awaitReadableSegment(request, timeoutMs) + } + return if (segment == null) { + SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) + } else { + stream(holder, request, segment, generation) + } + } + + private suspend fun fetchLive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + generation: Long, + ): SabrPlaybackSegmentResult { + sessionStore.cachedSegment(holder, request)?.let { + holder.markServed(it, generation) + return SabrPlaybackSegmentResult.Ready(it.mimeType, it.bytes) + } + sessionStore.requestSegmentDemand(holder, request, generation) + val segment = awaitCachedLive(holder, request, timeoutMs) + return if (segment == null) SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) else { + holder.clearSegmentDemand(request) + holder.markServed(segment, generation) + SabrPlaybackSegmentResult.Ready(segment.mimeType, segment.bytes) + } + } + + private suspend fun awaitCachedLive( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, + ): CachedSabrSegment? = withTimeoutOrNull(timeoutMs) { + var segment = sessionStore.cachedSegment(holder, request) + while (segment == null && holder.terminalFailure() == null && holder.networkFailure() == null) { + delay(SEGMENT_WAIT_MS) + segment = sessionStore.cachedSegment(holder, request) + if (segment == null) segment = followingLiveSegment(holder, request) + } + segment + } + + private suspend fun followingLiveSegment( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + ): CachedSabrSegment? { + val targetMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) + return sessionStore.findCachedPlaybackMediaAt( + holder = holder, + format = request.format, + targetMs = targetMs, + predictedSequence = request.sequenceNumber, + allowFollowing = true, + )?.takeUnless { + holder.failLivePlaybackDiscontinuity( + request.format, + targetMs, + it, + holder.lastServedSequence(request.format) != null, + ) + } + } + + private fun stream( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + segment: SabrMediaSegment, + generation: Long, + ): SabrPlaybackSegmentResult.Stream { + holder.clearSegmentDemand(request) + return SabrPlaybackSegmentResult.Stream(request.format.mimeType.orEmpty(), segment, holder, generation) + } + + private companion object { + const val REPOSITIONING = "repositioning" + const val SEGMENT_WAIT_MS = 250L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt index e3e123eb..62774b80 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt @@ -1,7 +1,15 @@ package dev.typetype.server.services +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment + internal sealed class SabrPlaybackSegmentResult { data class Ready(val mimeType: String, val bytes: ByteArray) : SabrPlaybackSegmentResult() + data class Stream( + val mimeType: String, + val segment: SabrMediaSegment, + val holder: SabrSessionHolder, + val generation: Long, + ) : SabrPlaybackSegmentResult() data class Retry(val holder: SabrSessionHolder, val status: String) : SabrPlaybackSegmentResult() data class Stale(val holder: SabrSessionHolder) : SabrPlaybackSegmentResult() data object InvalidSequence : SabrPlaybackSegmentResult() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index b3fb8190..6eb0d40d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services -import kotlinx.coroutines.delay import kotlinx.coroutines.withTimeoutOrNull import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { + private val mediaFetcher = SabrPlaybackMediaFetcher(sessionStore) suspend fun prepare( videoId: String, userId: String, @@ -145,48 +145,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS if (generation > activeGeneration) return SabrPlaybackSegmentResult.InvalidGeneration val request = SabrSegmentRequest.media(format, sequence) if (generation < activeGeneration) return staleMedia(holder, request) - sessionStore.cachedSegment(holder, request)?.let { - holder.markServed(it, generation) - return SabrPlaybackSegmentResult.Ready(it.mimeType, it.bytes) - } - sessionStore.requestSegmentDemand(holder, request, generation) - val segment = awaitCachedSegment(holder, request, timeoutMs) - return if (segment == null) SabrPlaybackSegmentResult.Retry(holder, REPOSITIONING) else { - holder.clearSegmentDemand(request) - holder.markServed(segment, generation) - SabrPlaybackSegmentResult.Ready(segment.mimeType, segment.bytes) - } - } - - private suspend fun awaitCachedSegment( - holder: SabrSessionHolder, - request: SabrSegmentRequest, - timeoutMs: Long, - ): CachedSabrSegment? = withTimeoutOrNull(timeoutMs) { - var segment = sessionStore.cachedSegment(holder, request) - while (segment == null && holder.terminalFailure() == null && holder.networkFailure() == null) { - delay(SEGMENT_WAIT_MS) - segment = sessionStore.cachedSegment(holder, request) - if (segment == null && holder.livePlaybackSnapshot()?.active == true) { - val targetMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) - val following = sessionStore.findCachedPlaybackMediaAt( - holder = holder, - format = request.format, - targetMs = targetMs, - predictedSequence = request.sequenceNumber, - allowFollowing = true, - ) - segment = following?.takeUnless { - holder.failLivePlaybackDiscontinuity( - request.format, - targetMs, - it, - holder.lastServedSequence(request.format) != null, - ) - } - } - } - segment + return mediaFetcher.fetch(holder, request, timeoutMs, generation) } private suspend fun staleMedia(holder: SabrSessionHolder, request: SabrSegmentRequest): SabrPlaybackSegmentResult = @@ -199,9 +158,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS private companion object { const val PREPARING = "preparing" - const val REPOSITIONING = "repositioning" const val INITIALIZATION_PRELOAD_TIMEOUT_MS = 6_000L - const val SEGMENT_WAIT_MS = 250L const val LIVE_INITIAL_PUMPS = 8 const val OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS = 9_007_199_254_740_991L } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index c9a759e2..d3230963 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -52,6 +52,21 @@ internal object SabrPumpLogger { ) } + fun expired(holder: SabrSessionHolder, request: SabrSegmentRequest, recoverable: Boolean): Unit { + logger.warn( + "sabr_pump event=demand_expired videoId={} request={} recoverable={} state={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", + holder.key.videoId, + request.summary(), + recoverable, + holder.playbackState(), + holder.session.requestNumber, + holder.session.streamState.getMinBufferedEndMs(), + holder.readerHeadMs(), + holder.readerTailMs(), + holder.session.cachedBytes, + ) + } + fun recovery(holder: SabrSessionHolder, action: SabrDemandRecoveryAction, request: SabrSegmentRequest): Unit { logger.info( "sabr_pump event=demand_recovery videoId={} request={} action={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt index 8c6a9a03..db315d9d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt @@ -11,6 +11,9 @@ internal class SabrSegmentCache { fun put(holder: SabrSessionHolder, segment: SabrMediaSegment): Unit { val format = holder.formatForItag(segment.header.itag) ?: return holder.observeMediaSegment(segment) + if (holder.key.purpose == SabrSessionPurpose.PLAYBACK && + !holder.expectsLive() && !segment.header.isInitSegment + ) return val mimeType = format.mimeType.orEmpty() val mediaParts = segment.takeIf { holder.expectsLive() && !it.header.isInitSegment } ?.let { SabrLiveMediaNormalizer.split(mimeType, it.data) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 1f3953ed..3a5fecd7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -183,7 +183,7 @@ internal class SabrSessionPumpLoop( if (holder.isHistoricalLiveRequest(request)) { holder.setPlaybackState(SabrPlaybackState.REPOSITIONING) holder.prepareForHistoricalLiveRewind(request) - return withTargetedRequestShape(holder, request) { + return withTargetedRequestShape(holder, request, prepareSession = false) { pumpUntilCached(holder, localization, request, runtime) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 5f666549..57f705e3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -151,7 +151,7 @@ internal class SabrSessionStore( internal suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) - + internal suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) internal suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) internal suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt index 0b14da86..6c7bbb77 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory internal inline fun withTargetedRequestShape( holder: SabrSessionHolder, request: SabrSegmentRequest, + prepareSession: Boolean = true, block: () -> T, ): T { val companion = holder.companionFormat(request.format) @@ -16,7 +17,7 @@ internal inline fun withTargetedRequestShape( val requestStartMs = holder.playbackSegmentStartMs(request.format, request.sequenceNumber) val targetPlayerTimeMs = request.targetPlayerTimeMs(holder, requestStartMs) val ranges = listOf(request.targetRange(holder), companion.targetCompanionRange(holder, targetPlayerTimeMs)) - holder.session.prepareForMediaSegment(request) + if (prepareSession) holder.session.prepareForMediaSegment(request) state.setPlayerTimeMs(targetPlayerTimeMs) state.setRequestTrackMode(request.trackMode(), true, true) state.setSelectVideoFormatBeforeAudio(request.format.isAudio) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 9529ad45..5c92769b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -9,6 +9,8 @@ import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.batchInsert import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll @@ -103,19 +105,31 @@ class SubscriptionGroupsService { userId: String, groupId: String, rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = addSubscriptions(userId, groupId, listOf(rawChannelUrl)) + + suspend fun addSubscriptions( + userId: String, + groupId: String, + rawChannelUrls: List, ): SubscriptionGroupMembershipResult = DatabaseFactory.query { SubscriptionMutationLock.acquire(userId) if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound - val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) - val subscriptionExists = SubscriptionsTable.selectAll().where { - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) - }.any() - if (!subscriptionExists) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound - SubscriptionGroupMembershipsTable.insertIgnore { - it[SubscriptionGroupMembershipsTable.groupId] = groupId - it[SubscriptionGroupMembershipsTable.userId] = userId - it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl - it[addedAt] = System.currentTimeMillis() + val channelUrls = rawChannelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl inList channelUrls) + }.mapTo(hashSetOf()) { it[SubscriptionsTable.channelUrl] } + if (subscribed.size != channelUrls.size) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound + val existing = SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList channelUrls) + }.mapTo(hashSetOf()) { it[SubscriptionGroupMembershipsTable.channelUrl] } + val addedAt = System.currentTimeMillis() + SubscriptionGroupMembershipsTable.batchInsert(channelUrls - existing, shouldReturnGeneratedValues = false) { url -> + this[SubscriptionGroupMembershipsTable.groupId] = groupId + this[SubscriptionGroupMembershipsTable.userId] = userId + this[SubscriptionGroupMembershipsTable.channelUrl] = url + this[SubscriptionGroupMembershipsTable.addedAt] = addedAt } SubscriptionGroupMembershipResult.Success } @@ -138,6 +152,22 @@ class SubscriptionGroupsService { } } + suspend fun removeSubscriptions( + userId: String, + groupId: String, + rawChannelUrls: List, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrls = rawChannelUrls.mapTo(hashSetOf(), ChannelUrlCanonicalizer::canonicalize) + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList channelUrls) + } + SubscriptionGroupMembershipResult.Success + } + suspend fun getChannelUrls(userId: String, groupId: String): List = DatabaseFactory.query { SubscriptionGroupMembershipsTable.selectAll().where { (SubscriptionGroupMembershipsTable.groupId eq groupId) and @@ -182,6 +212,7 @@ class SubscriptionGroupsService { companion object { const val MAX_GROUP_NAME_LENGTH = 100 + const val MAX_MEMBERSHIP_CHANNELS = 500 private const val UNIQUE_VIOLATION_SQL_STATE = "23505" } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 6e90ab8f..b0bb5dd9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -2,7 +2,9 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupMembershipItem import dev.typetype.server.models.SubscriptionItem import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder @@ -19,14 +21,39 @@ class SubscriptionsService { selection: SubscriptionSelection = SubscriptionSelection.All, ): List = DatabaseFactory.query { val selectedUrls = selectedChannelUrls(userId, selection) - val items = SubscriptionsTable.selectAll() - .where { SubscriptionsTable.userId eq userId } - .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) - .map { it.toItem() } + val selectedItems = subscriptionItems(userId) .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } - SubscriptionAvatarRepairer.repair(userId = userId, items = items) + SubscriptionAvatarRepairer.repair(userId = userId, items = selectedItems) } + suspend fun getAllWithGroupMemberships(userId: String): List = + DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + val groupIdsByChannel = SubscriptionGroupMembershipsTable + .innerJoin(SubscriptionGroupsTable) + .selectAll() + .where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupsTable.userId eq userId) + } + .groupBy( + keySelector = { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionGroupMembershipsTable.channelUrl]) + }, + valueTransform = { it[SubscriptionGroupMembershipsTable.groupId] }, + ) + .mapValues { (_, groupIds) -> groupIds.sorted() } + SubscriptionAvatarRepairer.repair(userId = userId, items = subscriptionItems(userId)).map { item -> + SubscriptionGroupMembershipItem( + channelUrl = item.channelUrl, + name = item.name, + avatarUrl = item.avatarUrl, + subscribedAt = item.subscribedAt, + groupIds = groupIdsByChannel[item.channelUrl].orEmpty(), + ) + } + } + suspend fun getChannelUrls(userId: String, selection: SubscriptionSelection): Set = DatabaseFactory.query { selectedChannelUrls(userId, selection) } @@ -55,6 +82,13 @@ class SubscriptionsService { SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0 } + private fun subscriptionItems(userId: String): List { + return SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq userId } + .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) + .map { it.toItem() } + } + private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { val all = SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } diff --git a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt index dcfbb3d7..283479f6 100644 --- a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt +++ b/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt @@ -18,3 +18,9 @@ internal data class TokenYoutubeSession( val isLiveContent: Boolean, val hlsUrl: String = "", ) + +internal fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { + val boundToken = token?.takeIf { it.visitorData == info.visitorData } ?: return null + return SabrPreparedInfo(info, boundToken, isLive, isLiveContent) + .takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index ae313935..962039b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -2,6 +2,8 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" @@ -9,17 +11,29 @@ internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access t internal class YoutubeSessionSabrStreamService( private val metadataService: YoutubeSessionStreamService, private val infoService: AuthenticatedSabrInfoService, + private val timeoutMs: Long = AuthenticatedSabrPolicy.STREAM_TIMEOUT_MS, ) { suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { - val metadata = metadataService.getStreamInfo(userId, url) ?: return null - if (metadata !is ExtractionResult.Success) return metadata - val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") - return when (val info = infoService.fetch(userId, videoId)) { - is AuthenticatedSabrInfoResult.Ready -> - ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) - AuthenticatedSabrInfoResult.Failed -> - ExtractionResult.Failure("Authenticated SABR playback unavailable") - AuthenticatedSabrInfoResult.NotConnected -> null + return try { + withTimeout(timeoutMs) { + val metadata = metadataService.getStreamInfo(userId, url) ?: return@withTimeout null + if (metadata !is ExtractionResult.Success) return@withTimeout metadata + val videoId = youtubeVideoId(url) ?: return@withTimeout ExtractionResult.BadRequest("Invalid YouTube URL") + when (val info = infoService.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> + ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) + AuthenticatedSabrInfoResult.Failed -> + ExtractionResult.Failure("Authenticated SABR playback unavailable") + AuthenticatedSabrInfoResult.TimedOut -> + ExtractionResult.Failure( + "Authenticated SABR preparation timed out", + AuthenticatedSabrPolicy.TIMEOUT_CODE, + ) + AuthenticatedSabrInfoResult.NotConnected -> null + } + } + } catch (error: TimeoutCancellationException) { + ExtractionResult.Failure("Authenticated SABR preparation timed out", AuthenticatedSabrPolicy.TIMEOUT_CODE) } } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt index 5c0c3c6f..ba8e0bea 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.ServiceList import java.util.concurrent.Semaphore import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean object YoutubeSessionTokenScope { private const val PUBLIC_PERMITS = 64 @@ -35,11 +37,18 @@ object YoutubeSessionTokenScope { } private suspend fun withPermits(count: Int, block: suspend () -> T): T { - withContext(Dispatchers.IO) { permits.acquire(count) } + val acquired = AtomicBoolean(false) return try { + withContext(Dispatchers.IO) { + if (!permits.tryAcquire(count, PERMIT_ACQUIRE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + error("Timed out waiting for YouTube extraction permits") + } + acquired.set(true) + } block() } finally { - permits.release(count) + if (acquired.get()) permits.release(count) } } + private const val PERMIT_ACQUIRE_TIMEOUT_MS = 15_000L } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index c14db660..4846bce3 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -20,13 +20,17 @@ object YoutubeTakeoutActivitySignalService { val normalized = entry.name.lowercase() if (entry.isDirectory || !normalized.endsWith(".html") || !normalized.contains("youtube")) return@forEach val html = zip.getInputStream(entry).bufferedReader().use { it.readText() }.replace("\u00a0", " ") - subscriptions += parseSubscriptions(html) - favorites += parseFavorites(html) + val parsed = parseHtml(html) + subscriptions += parsed.first + favorites += parsed.second } } return subscriptions.distinctBy { it.channelUrl } to favorites.distinctBy { it.videoUrl } } + internal fun parseHtml(html: String): Pair, List> = + parseSubscriptions(html) to parseFavorites(html) + private fun parseSubscriptions(html: String): List { return subscribedRegex.findAll(html).mapNotNull { match -> val url = channelUrlRegex.find(decode(match.groupValues[1]))?.value ?: return@mapNotNull null diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt index 5028a11a..988e3125 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt @@ -1,48 +1,87 @@ package dev.typetype.server.services import java.io.BufferedReader +import java.io.PushbackReader object YoutubeTakeoutCsvReader { fun parse(reader: BufferedReader): Pair, List>> { - val records = parseRecords(reader.readText()).filter { row -> row.any { it.isNotBlank() } } - if (records.isEmpty()) return emptyList() to emptyList() - val header = records.first().mapIndexed { index, value -> - if (index == 0) value.removePrefix("\uFEFF") else value + var header = emptyList() + val rows = mutableListOf>() + forEach(reader, { header = it }) { row -> + require(rows.size < MAX_COLLECTED_ROWS) { "CSV contains too many rows" } + rows += row } - return header to records.drop(1) + return header to rows } - private fun parseRecords(content: String): List> { - val records = mutableListOf>() + fun forEach( + reader: BufferedReader, + onHeader: (List) -> Unit, + onRow: (List) -> Unit, + ) { + var headerSeen = false + parseRecords(reader) { record -> + if (record.none(String::isNotBlank)) return@parseRecords + if (!headerSeen) { + onHeader(record.mapIndexed { index, value -> if (index == 0) value.removePrefix("\uFEFF") else value }) + headerSeen = true + } else { + onRow(record) + } + } + } + + private fun parseRecords(source: BufferedReader, block: (List) -> Unit) { + val reader = PushbackReader(source, 1) val row = mutableListOf() val current = StringBuilder() var quoted = false - var index = 0 - while (index < content.length) { - val c = content[index] + var hasData = false + while (true) { + val value = reader.read() + if (value == -1) break + val c = value.toChar() + hasData = true when { - c == '"' && quoted && content.getOrNull(index + 1) == '"' -> { - current.append(c) - index += 1 + c == '"' && quoted -> { + val next = reader.read() + if (next == '"'.code) current.append('"') else { + quoted = false + if (next != -1) reader.unread(next) + } } - c == '"' -> quoted = quoted.not() + c == '"' -> quoted = true c == ',' && !quoted -> { - row += current.toString().trim() - current.clear() + row.addField(current) } (c == '\n' || c == '\r') && !quoted -> { - if (c == '\r' && content.getOrNull(index + 1) == '\n') index += 1 - row += current.toString().trim() - records += row.toList() + if (c == '\r') { + val next = reader.read() + if (next != '\n'.code && next != -1) reader.unread(next) + } + row.addField(current) + block(row.toList()) row.clear() - current.clear() + hasData = false } else -> current.append(c) } - index += 1 + require(current.length <= MAX_FIELD_CHARS) { "CSV field is too large" } + require(row.size <= MAX_COLUMNS) { "CSV contains too many columns" } } - row += current.toString().trim() - records += row.toList() - return records + require(!quoted) { "CSV contains an unterminated quoted field" } + if (hasData || current.isNotEmpty() || row.isNotEmpty()) { + row.addField(current) + block(row.toList()) + } + } + + private fun MutableList.addField(value: StringBuilder) { + add(value.toString().trim()) + value.clear() } + + private const val MAX_COLUMNS = 256 + private const val MAX_FIELD_CHARS = 2 * 1024 * 1024 + private const val MAX_COLLECTED_ROWS = 100_000 } diff --git a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt b/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt index fa2bf1df..6cf607c4 100644 --- a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt @@ -82,6 +82,22 @@ class BugReportRoutesTest { assertEquals(HttpStatusCode.Created, response.status) } + @Test + fun `POST bug report accepts bounded device diagnostics`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { bugReportRoutes(service, auth) } + } + val response = client.post("/bug-reports") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + contentType(ContentType.Application.Json) + setBody( + """{"category":"ui","description":"Responsive issue","context":{"route":"/settings","timestamp":1774200000000,"userAgent":"Mozilla","browserLanguage":"fr-FR","viewportWidth":390,"viewportHeight":844,"screenWidth":430,"screenHeight":932,"devicePixelRatio":3.0,"online":true,"timezone":"Europe/Paris"}}""", + ) + } + assertEquals(HttpStatusCode.Created, response.status) + } + @Test fun `POST bug report with invalid category returns 400`() = testApplication { application { diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt index 0f4c544d..2e19b99d 100644 --- a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt @@ -4,6 +4,7 @@ import com.sun.net.httpserver.HttpServer import dev.typetype.server.routes.downloaderGatewayRoutes import dev.typetype.server.services.DownloaderGatewayService import io.ktor.client.request.get +import io.ktor.client.request.head import io.ktor.client.request.header import io.ktor.client.statement.bodyAsText import io.ktor.http.HttpHeaders @@ -20,6 +21,39 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class DownloaderGatewayArtifactProxyTest { + @Test + fun `direct artifact head preserves metadata without a response body`() = testApplication { + val requestedMethod = AtomicReference() + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + requestedMethod.set(exchange.requestMethod) + exchange.responseHeaders.add(HttpHeaders.ContentType, "video/mp4") + exchange.responseHeaders.add(HttpHeaders.ContentLength, "4096") + exchange.responseHeaders.add(HttpHeaders.AcceptRanges, "bytes") + exchange.responseHeaders.add(HttpHeaders.ETag, "\"artifact-v1\"") + exchange.responseHeaders.add(HttpHeaders.LastModified, "Sat, 22 Aug 2026 10:00:00 GMT") + exchange.sendResponseHeaders(200, -1) + exchange.close() + } + upstream.start() + application { + routing { downloaderGatewayRoutes(DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}")) } + } + + try { + val response = client.head("/downloader/jobs/test/artifact") + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("HEAD", requestedMethod.get()) + assertEquals("4096", response.headers[HttpHeaders.ContentLength]) + assertEquals("bytes", response.headers[HttpHeaders.AcceptRanges]) + assertEquals("\"artifact-v1\"", response.headers[HttpHeaders.ETag]) + assertEquals("Sat, 22 Aug 2026 10:00:00 GMT", response.headers[HttpHeaders.LastModified]) + assertEquals("", response.bodyAsText()) + } finally { + upstream.stop(0) + } + } + @Test fun `internal artifact redirect streams range response`() = testApplication { val requestedRange = AtomicReference() @@ -91,6 +125,32 @@ class DownloaderGatewayArtifactProxyTest { } } + @Test + fun `direct artifact preserves unsatisfied range response`() = testApplication { + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + exchange.responseHeaders.add(HttpHeaders.ContentRange, "bytes */6") + exchange.responseHeaders.add(HttpHeaders.AcceptRanges, "bytes") + exchange.sendResponseHeaders(416, -1) + exchange.close() + } + upstream.start() + application { + routing { downloaderGatewayRoutes(DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}")) } + } + + try { + val response = client.get("/downloader/jobs/test/artifact") { + header(HttpHeaders.Range, "bytes=20-30") + } + assertEquals(HttpStatusCode.fromValue(416), response.status) + assertEquals("bytes */6", response.headers[HttpHeaders.ContentRange]) + assertEquals("bytes", response.headers[HttpHeaders.AcceptRanges]) + } finally { + upstream.stop(0) + } + } + private fun testDns(): Dns = Dns { hostname -> if (hostname == "typetype-garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) } diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index 7d2eeb22..7eca4fc0 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -72,6 +72,7 @@ class InstanceRoutesTest { assertEquals(false, root["oidcEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) + assertEquals(false, root["parentalControlsEnabled"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) val rss = root["rss"]?.jsonObject assertEquals(false, rss?.get("enabled")?.jsonPrimitive?.boolean) @@ -93,6 +94,7 @@ class InstanceRoutesTest { localLoginEnabled = false, oidcAutoRedirect = true, youtubeRemoteLoginEnabled = true, + accessMode = "allow_list", rssEnabled = true, rssPublicBaseUrl = "https://video.example/", rssMaxFeedsPerUser = 4, @@ -123,6 +125,7 @@ class InstanceRoutesTest { assertEquals(true, root["oidcAutoRedirect"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) + assertEquals(true, root["parentalControlsEnabled"]?.jsonPrimitive?.boolean) assertEquals(null, root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) val rss = root["rss"]!!.jsonObject assertEquals(true, rss["enabled"]?.jsonPrimitive?.boolean) diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt new file mode 100644 index 00000000..babd1b5a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt @@ -0,0 +1,169 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.portability.NewPipePortabilityAdapter +import dev.typetype.server.portability.PortabilityCategory +import dev.typetype.server.portability.PortabilityDataPort +import dev.typetype.server.portability.PortabilityEngine +import dev.typetype.server.portability.PortabilityExportRequest +import dev.typetype.server.portability.PortabilityFormat +import dev.typetype.server.portability.PortabilityImportRequest +import dev.typetype.server.portability.PortabilityJobSnapshot +import dev.typetype.server.portability.PortabilityJobState +import dev.typetype.server.portability.PortabilityJobStore +import dev.typetype.server.portability.PortabilityRecordSink +import dev.typetype.server.portability.PortabilityRecordSource +import dev.typetype.server.portability.PortabilityRegistry +import dev.typetype.server.portability.PortabilitySubscription +import dev.typetype.server.routes.portabilityRoutes +import dev.typetype.server.services.AuthService +import io.ktor.client.call.body +import io.ktor.client.request.delete +import io.ktor.client.request.forms.MultiPartFormDataContent +import io.ktor.client.request.forms.formData +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.Headers +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityRoutesTest { + @TempDir + lateinit var directory: Path + + @Test + fun `authenticated account can preview apply export and delete`() = testApplication { + val engine = engine() + application { + installRequestObservability() + install(ContentNegotiation) { json(CacheJson) } + routing { portabilityRoutes(engine, AuthService.fixed("owner")) } + } + + val formats = client.get("/portability/formats") { authorize() } + assertEquals(HttpStatusCode.OK, formats.status) + + val upload = client.post("/portability/imports") { + authorize() + header(REQUEST_ID_HEADER, "portability-route-request") + setBody( + MultiPartFormDataContent( + formData { + append( + "file", + """{"subscriptions":[],"app_version":"0.28.0"}""".toByteArray(), + Headers.build { + append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + append(HttpHeaders.ContentDisposition, "filename=newpipe.json") + }, + ) + }, + ), + ) + } + assertEquals(HttpStatusCode.Accepted, upload.status) + val uploaded = upload.snapshot() + assertEquals("portability-route-request", uploaded.requestId) + val importId = uploaded.id + val preview = awaitState(importId, PortabilityJobState.READY) + assertEquals(0L, preview.preview?.counts?.get("subscriptions")) + assertEquals(0L, preview.progress?.processed) + + val apply = client.post("/portability/jobs/$importId/apply") { + authorize() + contentType(ContentType.Application.Json) + setBody(CacheJson.encodeToString(PortabilityImportRequest.serializer(), PortabilityImportRequest(setOf(PortabilityCategory.SUBSCRIPTIONS)))) + } + assertEquals(HttpStatusCode.Accepted, apply.status) + awaitState(importId, PortabilityJobState.COMPLETED) + + val export = client.post("/portability/exports") { + authorize() + contentType(ContentType.Application.Json) + setBody( + CacheJson.encodeToString( + PortabilityExportRequest.serializer(), + PortabilityExportRequest(PortabilityFormat.NEW_PIPE, setOf(PortabilityCategory.SUBSCRIPTIONS)), + ), + ) + } + assertEquals(HttpStatusCode.Accepted, export.status) + val exportId = export.snapshot().id + awaitState(exportId, PortabilityJobState.COMPLETED) + val report = client.get("/portability/jobs/$exportId/report") { authorize() } + assertEquals(HttpStatusCode.OK, report.status) + val artifact = client.get("/portability/jobs/$exportId/artifact") { authorize() } + assertEquals(HttpStatusCode.OK, artifact.status) + assertEquals(listOf(0x50, 0x4b), artifact.body().take(2).map(Byte::toInt)) + + val deleted = client.delete("/portability/jobs/$importId") { authorize() } + assertEquals(HttpStatusCode.NoContent, deleted.status) + engine.close() + } + + private fun engine() = PortabilityEngine( + PortabilityRegistry(listOf(NewPipePortabilityAdapter())), + RouteDataPort, + PortabilityJobStore(directory.resolve("jobs")), + CoroutineScope(SupervisorJob() + Dispatchers.Default), + ) + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() = + header(HttpHeaders.Authorization, "Bearer test-jwt") + + private suspend fun io.ktor.client.statement.HttpResponse.snapshot(): PortabilityJobSnapshot = + CacheJson.decodeFromString(PortabilityJobSnapshot.serializer(), bodyAsText()) + + private suspend fun io.ktor.server.testing.ApplicationTestBuilder.awaitState( + id: String, + expected: PortabilityJobState, + ): PortabilityJobSnapshot { + repeat(100) { + val response = client.get("/portability/jobs/$id") { authorize() } + val snapshot = CacheJson.decodeFromString(PortabilityJobSnapshot.serializer(), response.bodyAsText()) + if (snapshot.state == expected) return snapshot + delay(10) + } + error("Job did not reach $expected") + } +} + +private object RouteDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) = source.counts().mapKeys { it.key.wireName }.also { result -> + request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1", "One")) + onCategoryComplete(PortabilityCategory.SUBSCRIPTIONS, 1L) + } +} diff --git a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt b/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt index f11a527e..8d352240 100644 --- a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.routes.respondSabrMediaBytes +import dev.typetype.server.routes.respondSabrMediaStream import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.statement.bodyAsBytes @@ -13,6 +14,8 @@ import io.ktor.server.testing.testApplication import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.util.concurrent.atomic.AtomicInteger class SabrMediaResponseWriterTest { @Test @@ -40,11 +43,33 @@ class SabrMediaResponseWriterTest { assertArrayEquals(byteArrayOf(2, 3, 4, 5), response.bodyAsBytes()) } - private fun ApplicationTestBuilder.installApp(): Unit = application { + @Test + fun `sabr media stream honors byte ranges without buffering the body`() = testApplication { + val opened = AtomicInteger() + installApp(opened) + + val response = client.get("/stream") { header(HttpHeaders.Range, "bytes=3-7") } + + assertEquals(HttpStatusCode.PartialContent, response.status) + assertEquals("bytes 3-7/10", response.headers[HttpHeaders.ContentRange]) + assertEquals("5", response.headers[HttpHeaders.ContentLength]) + assertArrayEquals(byteArrayOf(3, 4, 5, 6, 7), response.bodyAsBytes()) + assertEquals(1, opened.get()) + } + + private fun ApplicationTestBuilder.installApp(opened: AtomicInteger = AtomicInteger()): Unit = application { routing { get("/media") { call.respondSabrMediaBytes("video/mp4; codecs=\"avc1.640028\"", ByteArray(10) { it.toByte() }) } + get("/stream") { + call.respondSabrMediaStream( + "video/mp4; codecs=\"avc1.640028\"", + 10L, + { ByteArrayInputStream(ByteArray(10) { it.toByte() }) }, + opened::incrementAndGet, + ) + } } } } diff --git a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt index 66e4e649..75475493 100644 --- a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt @@ -119,7 +119,7 @@ class SabrPlaybackGranularRoutesTest { } @Test - fun `window queues missing first video without direct fetch`() = testApplication { + fun `window advertises missing first video without direct fetch`() = testApplication { val store = audioOnlyStore() val holder = holder() every { store.lookupByToken("session-token") } returns holder @@ -130,13 +130,13 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody()) } - assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(response.bodyAsText().contains("video:136:1 pending")) + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().contains("segment/1")) verify(atLeast = 1) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } } @Test - fun `window queues video first when blocked tracks have equal coverage`() = testApplication { + fun `window keeps stale timeline pending and queues both tracks`() = testApplication { val store = emptyStore() val holder = holder() every { store.lookupByToken("session-token") } returns holder @@ -147,9 +147,7 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody(340_000L)) } - val body = response.bodyAsText() assertEquals(HttpStatusCode.Accepted, response.status) - assertTrue(body.contains("video:136:1 pending")) verify(exactly = 2) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } coVerify(exactly = 0) { store.fetchInitializationData(holder, any()) } } @@ -232,6 +230,7 @@ class SabrPlaybackGranularRoutesTest { val state = mockk() every { session.streamState } returns state every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null every { session.isBeyondEnd(any()) } returns false every { state.setActiveTrackTypes(true, true) } returns Unit every { state.setPlaybackRate(any()) } returns Unit diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt new file mode 100644 index 00000000..1b599dec --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupMembershipRequestLimitsRoutesTest.kt @@ -0,0 +1,132 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionGroupsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.OutgoingContent +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.ktor.utils.io.ByteWriteChannel +import io.ktor.utils.io.writeByteArray +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupMembershipRequestLimitsRoutesTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + private const val EXPECTED_BODY_LIMIT_BYTES = 1024 * 1024 + private const val EXPECTED_CHANNEL_URL_LIMIT = 2048 + + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `membership mutation rejects declared body over one mebibyte`() = withApp { + val group = createGroup() + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("x".repeat(EXPECTED_BODY_LIMIT_BYTES + 1)) + } + + assertEquals(HttpStatusCode.PayloadTooLarge, response.status) + } + + @Test + fun `membership mutation rejects streamed body over one mebibyte`() = withApp { + val group = createGroup() + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorize() + setBody(oversizedStreamingBody()) + } + + assertEquals(HttpStatusCode.PayloadTooLarge, response.status) + } + + @Test + fun `membership mutation rejects channel urls over 2048 characters`() = withApp { + val group = createGroup() + val channelUrl = "https://example.com/" + "a".repeat(EXPECTED_CHANNEL_URL_LIMIT) + + val response = client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"$channelUrl"}""") + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `membership deletion rejects query and whitespace body together`() = withApp { + val channelUrl = "https://example.com/channel" + subscriptions.add(TEST_USER_ID, SubscriptionItem(channelUrl, "Channel", "")) + val group = createGroup() + groups.addSubscription(TEST_USER_ID, group.id, channelUrl) + + val response = client.delete("/subscriptions/groups/${group.id}/channels") { + authorize() + parameter("url", channelUrl) + header(HttpHeaders.ContentType, ContentType.Text.Plain.toString()) + setBody(" ") + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertEquals(1, groups.getAll(TEST_USER_ID).single().channelCount) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionGroupsRoutes(groups, auth) } + } + block() + } + + private suspend fun createGroup() = requireNotNull( + (groups.create(TEST_USER_ID, "Work") as? SubscriptionGroupWriteResult.Success)?.group, + ) + + private fun oversizedStreamingBody() = object : OutgoingContent.WriteChannelContent() { + override val contentType = ContentType.Application.Json + + override suspend fun writeTo(channel: ByteWriteChannel) { + val chunk = ByteArray(64 * 1024) { 'x'.code.toByte() } + repeat(EXPECTED_BODY_LIMIT_BYTES / chunk.size + 1) { channel.writeByteArray(chunk) } + } + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorizeJson() { + authorize() + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt index 70dd04b3..8ba7517c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -6,6 +6,7 @@ import dev.typetype.server.routes.subscriptionGroupsRoutes import dev.typetype.server.routes.subscriptionsRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.delete import io.ktor.client.request.get @@ -25,6 +26,9 @@ import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -37,6 +41,8 @@ class SubscriptionGroupsRoutesTest { private val auth = AuthService.fixed(TEST_USER_ID) companion object { + private const val FOREIGN_USER_ID = "foreign-user" + @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() @@ -61,6 +67,11 @@ class SubscriptionGroupsRoutesTest { assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/groups").status) } + @Test + fun `group membership projection requires authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/group-memberships").status) + } + @Test fun `groups can be created listed renamed and deleted`() = withApp { val create = client.post("/subscriptions/groups") { @@ -121,6 +132,108 @@ class SubscriptionGroupsRoutesTest { assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) } + @Test + fun `membership routes add and remove multiple channels atomically`() = withApp { + val first = channel("one") + val second = channel("two") + val neverAdded = channel("three") + listOf(first, second, neverAdded).forEach { url -> + subscriptions.add(TEST_USER_ID, SubscriptionItem(url, url.substringAfterLast('/'), "")) + } + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second","$first"]}""") + }.status) + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second"]}""") + }.status) + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText() + assertTrue(grouped.contains(first)) + assertTrue(grouped.contains(second)) + assertTrue(!grouped.contains(neverAdded)) + + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$first","$second","$neverAdded"]}""") + }.status) + assertEquals("[]", authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText()) + } + + @Test + fun `membership deletion rejects query and body together`() = withApp { + val first = channel("one") + val second = channel("two") + listOf(first, second).forEach { url -> + subscriptions.add(TEST_USER_ID, SubscriptionItem(url, url.substringAfterLast('/'), "")) + } + val group = createGroup("Work") + listOf(first, second).forEach { url -> groups.addSubscription(TEST_USER_ID, group.id, url) } + + assertEquals(HttpStatusCode.BadRequest, client.delete("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + parameter("url", first) + setBody("""{"channelUrls":["$second"]}""") + }.status) + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText() + assertTrue(grouped.contains(first)) + assertTrue(grouped.contains(second)) + } + + @Test + fun `bulk membership addition changes nothing when a subscription is missing`() = withApp { + val subscribed = channel("subscribed") + subscriptions.add(TEST_USER_ID, SubscriptionItem(subscribed, "Subscribed", "")) + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NotFound, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrls":["$subscribed","${channel("missing")}"]}""") + }.status) + assertEquals("[]", authorizedGet("/subscriptions") { parameter("groupId", group.id) }.bodyAsText()) + } + + @Test + fun `group membership projection returns account scoped memberships with subscription data`() = withApp { + val sharedChannel = channel("shared") + subscriptions.add(TEST_USER_ID, SubscriptionItem(sharedChannel, "Shared", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("ungrouped"), "Ungrouped", "")) + subscriptions.add(FOREIGN_USER_ID, SubscriptionItem(sharedChannel, "Foreign shared", "")) + + val ownGroups = listOf(createGroup("Own"), createGroup("Another")) + ownGroups.forEach { group -> + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"$sharedChannel"}""") + }.status) + } + val foreignGroup = requireNotNull( + (groups.create(FOREIGN_USER_ID, "Foreign") as? SubscriptionGroupWriteResult.Success)?.group, + ) + groups.addSubscription(FOREIGN_USER_ID, foreignGroup.id, sharedChannel) + + val response = authorizedGet("/subscriptions/group-memberships") + assertEquals(HttpStatusCode.OK, response.status) + val items = Json.parseToJsonElement(response.bodyAsText()).jsonArray.map { it.jsonObject } + assertEquals(2, items.size) + + val shared = items.single { it.getValue("channelUrl").jsonPrimitive.content == sharedChannel } + assertEquals("Shared", shared.getValue("name").jsonPrimitive.content) + assertEquals("avatar", shared.getValue("avatarUrl").jsonPrimitive.content) + assertTrue(shared.getValue("subscribedAt").jsonPrimitive.content.toLong() > 0) + assertEquals( + ownGroups.map { it.id }.sorted(), + shared.getValue("groupIds").jsonArray.map { it.jsonPrimitive.content }, + ) + + val ungrouped = items.single { + it.getValue("channelUrl").jsonPrimitive.content == channel("ungrouped") + } + assertEquals(emptyList(), ungrouped.getValue("groupIds").jsonArray.map { it.jsonPrimitive.content }) + } + @Test fun `invalid or inaccessible filters fail explicitly`() = withApp { assertEquals(HttpStatusCode.BadRequest, authorizedGet("/subscriptions") { diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index 4233deae..7f017a97 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -212,6 +212,40 @@ class SubscriptionGroupsServiceTest { assertTrue(allWaited, "all group mutations must wait for the account-scoped lock") } + @Test + fun `membership projection shares the account subscription lock`() = runTest { + val userId = "projection-user" + val group = groups.create(userId, "Group").createdGroup() + subscriptions.add(userId, subscription("one")) + groups.addSubscription(userId, group.id, channel("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val projection = async(Dispatchers.IO) { subscriptions.getAllWithGroupMemberships(userId) } + val readWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + while (!projection.isCompleted && waitingSubscriptionLocks(userId) == 0) yield() + !projection.isCompleted + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assertTrue(readWaited, "the membership projection must wait for the account-scoped lock") + assertEquals(listOf(group.id), projection.await().single().groupIds) + } + @Test fun `replacement imports retain only memberships for subscriptions still present`() = runTest { val group = groups.create("user", "Group").createdGroup() diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index d0f988c8..de4341ef 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -3,12 +3,19 @@ package dev.typetype.server import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.FavoritesTable import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.SubscriptionAvatarRepairer +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SubscriptionsService import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -16,6 +23,7 @@ import org.junit.jupiter.api.Test class SubscriptionsAvatarRepairServiceTest { private val service = SubscriptionsService() + private val groups = SubscriptionGroupsService() companion object { const val WATCH_CHANNEL_URL = "https://www.youtube.com/channel/UCWatch" @@ -62,6 +70,28 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(26, second.count { it.avatarUrl.isNotBlank() }) } + @Test + fun `filtered getAll repairs avatars only for selected subscriptions`() = runTest { + repeat(25) { index -> + val channelUrl = "https://www.youtube.com/channel/UCUnselected$index" + addSubscription(channelUrl) + addHistory( + channelUrl = channelUrl, + avatarUrl = "https://avatar.test/unselected-$index.jpg", + watchedAt = (index + 1).toLong(), + ) + } + addSubscription(WATCH_CHANNEL_URL) + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL, watchedAt = 0) + val group = (groups.create(TEST_USER_ID, "Selected") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, WATCH_CHANNEL_URL) + + val selected = service.getAll(TEST_USER_ID, SubscriptionSelection.Group(group.id)).single() + + assertEquals(WATCH_AVATAR_URL, selected.avatarUrl) + assertEquals("", storedAvatar("https://www.youtube.com/channel/UCUnselected0")) + } + @Test fun `avatar repair scans past unrepairable empty subscriptions`() = runTest { addWatchLater(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL) @@ -78,6 +108,12 @@ class SubscriptionsAvatarRepairServiceTest { service.add(TEST_USER_ID, SubscriptionItem(channelUrl = channelUrl, name = "Channel", avatarUrl = "")) } + private suspend fun storedAvatar(channelUrl: String): String = DatabaseFactory.query { + SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq TEST_USER_ID) and (SubscriptionsTable.channelUrl eq channelUrl) + }.single()[SubscriptionsTable.avatarUrl] + } + private suspend fun addWatchLater(channelUrl: String, avatarUrl: String): Unit = DatabaseFactory.query { WatchLaterTable.insert { it[userId] = TEST_USER_ID; it[url] = "https://video.test/watch"; it[title] = "Video"; it[thumbnail] = "" diff --git a/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt new file mode 100644 index 00000000..dfd5541e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt @@ -0,0 +1,54 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class FlowPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves supported flow version two data`() { + val file = directory.resolve("flow.json") + Files.writeString( + file, + """ + { + "version":2,"timestamp":123, + "viewHistory":[{"videoId":"v1","position":12000,"duration":60000,"timestamp":10,"title":"One","channelId":"UC1"}], + "searchHistory":[{"query":"query","timestamp":11,"type":"TEXT"}], + "subscriptions":[{"channelId":"UC1","channelName":"Channel","channelThumbnail":"avatar","subscribedAt":12}], + "playlists":[{"id":"p1","name":"Saved","description":"List","createdAt":13}], + "playlistVideos":[{"playlistId":"p1","videoId":"v1","position":0,"addedAt":14}], + "videos":[{"id":"v1","title":"One","channelId":"UC1"}], + "subscriptionGroups":[{"name":"News","channelIds":"UC1","sortOrder":0}], + "likedVideos":[{"videoId":"v1","title":"One","likedAt":15}], + "contentPreferences":{"blockedChannels":["UC2"],"preferredTopics":[],"blockedTopics":[]} + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = FlowPortabilityAdapter() + + assertEquals(PortabilityFormat.FLOW, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories() - PortabilityCategory.PLAYLISTS) + assertTrue(output.toString().contains("\"version\":2")) + assertTrue(output.toString().contains("\"channelIds\":\"UC1\"")) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt new file mode 100644 index 00000000..98a3e439 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt @@ -0,0 +1,81 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream +import kotlin.io.path.outputStream + +class GrayjayPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports current grayjay reconstruction stores`() { + val archive = directory.resolve("grayjay.zip") + writeGrayjayFixture(archive) + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = GrayjayPortabilityAdapter() + + assertEquals("1", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + spool.delete() + } + + @Test + fun `exports a grayjay archive that round trips through the adapter`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://www.youtube.com/channel/UC1", "Channel")) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://www.youtube.com/channel/UC1")) + source.write(PortabilityHistory(PortabilityVideo("https://youtu.be/v1", "Title"), 100, 25)) + source.write(PortabilityPlaylist("p1", "Playlist")) + source.write(PortabilityPlaylistVideo("p1", 0, PortabilityVideo("https://youtu.be/v1"))) + source.write(PortabilityWatchLater(PortabilityVideo("https://youtu.be/v2"))) + val archive = directory.resolve("export.zip") + archive.outputStream().use { GrayjayPortabilityAdapter().encode(source, it, source.categories()) } + + ZipFile(archive.toFile()).use { zip -> + assertTrue(zip.getInputStream(zip.getEntry("exportInfo")).reader().readText().contains("\"version\":\"1\"")) + assertTrue(zip.getInputStream(zip.getEntry("stores/playlists")).reader().readText().contains("Playlist:::p1")) + } + val restored = PortabilitySpool.create(directory) + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + GrayjayPortabilityAdapter().decode(input, restored) + assertEquals(2L, restored.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(2L, restored.counts()[PortabilityCategory.PLAYLISTS]) + source.delete() + restored.delete() + } + + private fun writeGrayjayFixture(path: Path) { + ZipOutputStream(path.outputStream()).use { zip -> + entry(zip, "exportInfo", "{\"version\":\"1\"}") + entry(zip, "stores/subscriptions", "[\"https://www.youtube.com/channel/UC1\"]") + entry(zip, "stores/subscription_groups", "[\"{\\\"name\\\":\\\"News\\\",\\\"urls\\\":[\\\"https://www.youtube.com/channel/UC1\\\"]}\"]") + entry(zip, "stores/history", "[\"https://youtu.be/v1|||100|||25|||Title\"]") + entry(zip, "stores/playlists", "[\"Playlist:::p1\\nhttps://youtu.be/v1\\nhttps://youtu.be/v2\"]") + entry(zip, "stores/watch_later", "[\"https://youtu.be/v2\"]") + entry(zip, "plugins", "{}") + entry(zip, "plugin_settings", "{}") + } + } + + private fun entry(zip: ZipOutputStream, name: String, value: String) { + zip.putNextEntry(ZipEntry(name)) + zip.write(value.toByteArray()) + zip.closeEntry() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt new file mode 100644 index 00000000..cba31de5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class InvidiousPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves invidious account categories`() { + val file = directory.resolve("invidious.json") + Files.writeString( + file, + """{"subscriptions":["UC1"],"watch_history":["video000001"],"preferences":{"thin_mode":true},"playlists":[{"title":"Saved","description":"Keep","privacy":"Private","videos":["video000002"]}]}""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = InvidiousPortabilityAdapter() + + assertEquals(PortabilityFormat.INVIDIOUS, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.SETTINGS]) + assertEquals("missing_history_dates", spool.issues().single().code) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + val json = output.toString() + assertTrue(json.contains("\"subscriptions\":[\"UC1\"]")) + assertTrue(json.contains("\"title\":\"Saved\"")) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt new file mode 100644 index 00000000..46eca457 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class LibreTubePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves libretube account data`() { + val file = directory.resolve("libretube.json") + Files.writeString( + file, + """ + { + "format":"Piped","version":1, + "localSubscriptions":[{"channelId":"UC1","name":"One","avatar":"avatar"}], + "watchHistory":[{"videoId":"v1","title":"Video","duration":60}], + "watchPositions":[{"videoId":"v1","position":12000}], + "searchHistory":[{"query":"query"}], + "playlistBookmarks":[{"playlistId":"PL1","playlistName":"Saved","videos":2}], + "localPlaylists":[{"playlist":{"id":1,"name":"Local"},"videos":[{"videoId":"v1","title":"Video"}]}] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = LibreTubePortabilityAdapter() + + assertEquals(PortabilityFormat.LIBRE_TUBE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + assertTrue(output.toString().contains("\"localSubscriptions\"")) + assertTrue(output.toString().contains("\"watchPositions\"")) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt new file mode 100644 index 00000000..f62491b8 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class MaterialiousPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports and exports materialious invidious subscription json`() { + val file = directory.resolve("materialious-export.json") + Files.writeString(file, """{"subscriptions":["UC1","UC2"]}""") + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val adapter = MaterialiousPortabilityAdapter() + val spool = PortabilitySpool.create(directory) + + assertEquals("invidious-subscriptions", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("\"subscriptions\":[\"UC1\",\"UC2\"]")) + spool.delete() + } + + @Test + fun `accepts the opml format supported by materialious`() { + val file = directory.resolve("materialious.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + val adapter = MaterialiousPortabilityAdapter() + + assertEquals("opml", requireNotNull(adapter.detect(input)).formatVersion) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + spool.delete() + } + + @Test + fun `generic opml auto detection is not ambiguous with materialious`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val registry = PortabilityRegistry(listOf(OpmlPortabilityAdapter(), MaterialiousPortabilityAdapter())) + + assertEquals(PortabilityFormat.OPML, registry.detect(input).second.format) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt new file mode 100644 index 00000000..07226334 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityAdapterTest.kt @@ -0,0 +1,140 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.sql.DriverManager +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream +import java.util.zip.ZipOutputStream + +class NewPipeArchivePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `newpipe archive streams all supported tables`() { + val input = input(9) + val adapter = NewPipePortabilityAdapter() + val spool = PortabilitySpool.create(directory) + + assertEquals(PortabilityFormat.NEW_PIPE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + assertEquals(1L, spool.counts()[PortabilityCategory.SEARCH_HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.SAVED_PLAYLISTS]) + spool.delete() + } + + @Test + fun `pipepipe archive is distinguished by schema version`() { + val input = input(901) + val adapter = PipePipePortabilityAdapter() + + assertEquals(PortabilityFormat.PIPE_PIPE, requireNotNull(adapter.detect(input)).format) + assertEquals(null, NewPipePortabilityAdapter().detect(input)) + } + + @Test + fun `pipepipe export round trips canonical records through its real archive`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://youtube.com/channel/UC1", "One")) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://youtube.com/channel/UC1")) + val video = PortabilityVideo("https://youtube.com/watch?v=video000001", "Video", durationSeconds = 60) + source.write(PortabilityHistory(video, 10)) + source.write(PortabilityPlaylist("local", "Local")) + source.write(PortabilityPlaylistVideo("local", 0, video)) + source.write(PortabilityProgress(video.url, 20)) + source.write(PortabilitySearchHistory("query", 30)) + source.write(PortabilitySavedPlaylist("remote", "https://youtube.com/playlist?list=PL1", "Remote")) + val output = directory.resolve("pipepipe-export.zip") + val categories = source.categories() + + Files.newOutputStream(output).use { + PipePipePortabilityAdapter().encode(source, it, categories) + } + assertValidRoomArchive(output, NewPipeArchiveTarget.PIPE_PIPE) + val input = PortabilityInputFactory.create(output, output.fileName.toString(), "application/zip") + assertEquals("901", PipePipePortabilityAdapter().detect(input)?.formatVersion) + val restored = PortabilitySpool.create(directory) + PipePipePortabilityAdapter().decode(input, restored) + + assertEquals(1L, restored.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, restored.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, restored.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, restored.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, restored.counts()[PortabilityCategory.PROGRESS]) + assertEquals(1L, restored.counts()[PortabilityCategory.SEARCH_HISTORY]) + assertEquals(1L, restored.counts()[PortabilityCategory.SAVED_PLAYLISTS]) + source.delete() + restored.delete() + } + + private fun assertValidRoomArchive(archive: Path, target: NewPipeArchiveTarget) { + val database = directory.resolve("validated-${target.databaseVersion}.db") + ZipInputStream(Files.newInputStream(archive)).use { zip -> + check(zip.nextEntry?.name == "newpipe.db") + Files.newOutputStream(database).use(zip::copyTo) + } + DriverManager.getConnection("jdbc:sqlite:$database").use { sqlite -> + sqlite.createStatement().use { statement -> + statement.executeQuery("PRAGMA user_version").use { + assertEquals(target.databaseVersion, it.getInt(1)) + } + statement.executeQuery("PRAGMA integrity_check").use { + assertEquals("ok", it.getString(1)) + } + statement.executeQuery("PRAGMA foreign_key_check").use { + assertEquals(false, it.next()) + } + statement.executeQuery("SELECT identity_hash FROM room_master_table WHERE id = 42").use { + assertEquals(target.identityHash, it.getString(1)) + } + } + } + } + + private fun input(version: Int): PortabilityInput { + val db = directory.resolve("source-$version.db") + DriverManager.getConnection("jdbc:sqlite:$db").use { sqlite -> + sqlite.createStatement().use { statement -> + statement.execute("PRAGMA user_version = $version") + statement.execute("CREATE TABLE subscriptions(uid INTEGER, service_id INTEGER, url TEXT, name TEXT, avatar_url TEXT)") + statement.execute("CREATE TABLE streams(uid INTEGER, url TEXT, title TEXT, duration INTEGER, uploader TEXT, uploader_url TEXT, thumbnail_url TEXT)") + statement.execute("CREATE TABLE stream_history(stream_id INTEGER, access_date INTEGER)") + statement.execute("CREATE TABLE stream_state(stream_id INTEGER, progress_time INTEGER)") + statement.execute("CREATE TABLE playlists(uid INTEGER, name TEXT, display_index INTEGER)") + statement.execute("CREATE TABLE playlist_stream_join(playlist_id INTEGER, stream_id INTEGER, join_index INTEGER)") + statement.execute("CREATE TABLE remote_playlists(uid INTEGER, url TEXT, name TEXT, thumbnail_url TEXT, uploader TEXT, stream_count INTEGER, display_index INTEGER)") + statement.execute("CREATE TABLE feed_group(uid INTEGER, name TEXT, sort_order INTEGER)") + statement.execute("CREATE TABLE feed_group_subscription_join(group_id INTEGER, subscription_id INTEGER)") + statement.execute("CREATE TABLE search_history(id INTEGER, search TEXT, creation_date INTEGER)") + statement.execute("INSERT INTO subscriptions VALUES (1, 0, 'https://youtube.com/channel/UC1', 'One', 'avatar')") + statement.execute("INSERT INTO streams VALUES (1, 'https://youtube.com/watch?v=v1', 'Video', 60, 'One', 'https://youtube.com/channel/UC1', 'thumb')") + statement.execute("INSERT INTO stream_history VALUES (1, 10)") + statement.execute("INSERT INTO stream_state VALUES (1, 20)") + statement.execute("INSERT INTO playlists VALUES (1, 'Local', 0)") + statement.execute("INSERT INTO playlist_stream_join VALUES (1, 1, 0)") + statement.execute("INSERT INTO remote_playlists VALUES (2, 'https://youtube.com/playlist?list=PL1', 'Remote', 'thumb', 'One', 1, 0)") + statement.execute("INSERT INTO feed_group VALUES (1, 'News', 0)") + statement.execute("INSERT INTO feed_group_subscription_join VALUES (1, 1)") + statement.execute("INSERT INTO search_history VALUES (1, 'query', 30)") + } + } + val archive = directory.resolve("backup-$version.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("newpipe.db")) + Files.newInputStream(db).use { it.copyTo(output) } + output.closeEntry() + } + return PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt new file mode 100644 index 00000000..a64d2213 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class NewPipePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter streams supported subscriptions and reports unsupported services`() { + val file = directory.resolve("newpipe.json") + Files.writeString( + file, + """{"subscriptions":[{"service_id":0,"url":"https://www.youtube.com/channel/UC1","name":"One"},{"service_id":1,"url":"https://soundcloud.com/two","name":"Two"}],"app_version":"0.28.0","app_version_int":1000}""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = NewPipePortabilityAdapter() + + assertEquals(PortabilityFormat.NEW_PIPE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 1L), spool.counts()) + assertEquals("unsupported_subscription_provider", spool.issues().single().code) + + val output = directory.resolve("export.zip") + Files.newOutputStream(output).use { adapter.encode(spool, it, setOf(PortabilityCategory.SUBSCRIPTIONS)) } + val restored = PortabilitySpool.create(directory) + adapter.decode(PortabilityInputFactory.create(output, "export.zip", "application/zip"), restored) + assertEquals(1L, restored.counts()[PortabilityCategory.SUBSCRIPTIONS]) + restored.delete() + spool.delete() + } + + @Test + fun `adapter preserves an empty subscription section`() { + val file = directory.resolve("empty.json") + Files.writeString(file, """{"subscriptions":[],"app_version":"0.28.0"}""") + val spool = PortabilitySpool.create(directory) + + NewPipePortabilityAdapter().decode( + PortabilityInputFactory.create(file, file.fileName.toString(), "application/json"), + spool, + ) + + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 0L), spool.counts()) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt new file mode 100644 index 00000000..2f8d4aa2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt @@ -0,0 +1,65 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class OpmlPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter imports feeds and writes portable opml`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + val adapter = OpmlPortabilityAdapter() + + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("channel_id=UC1")) + spool.delete() + } + + @Test + fun `adapter does not resolve external entities`() { + val file = directory.resolve("unsafe.opml") + Files.writeString(file, """]>""") + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + val spool = PortabilitySpool.create(directory) + + assertThrows(Exception::class.java) { OpmlPortabilityAdapter().decode(input, spool) } + spool.delete() + } + + @Test + fun `skytube and youtube local use their supported opml exchange`() { + val file = directory.resolve("subscriptions.opml") + Files.writeString( + file, + """""", + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/xml") + listOf(PortabilityFormat.SKY_TUBE, PortabilityFormat.YOUTUBE_LOCAL).forEach { format -> + val adapter = OpmlPortabilityAdapter(format, autoDetect = false) + val spool = PortabilitySpool.create(directory) + assertEquals(format, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + val output = ByteArrayOutputStream() + adapter.encode(spool, output, setOf(PortabilityCategory.SUBSCRIPTIONS)) + assertTrue(output.toString().contains("channel_id=UC1")) + spool.delete() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt new file mode 100644 index 00000000..d92af2b6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt @@ -0,0 +1,47 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class PipedPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter preserves piped account categories`() { + val file = directory.resolve("piped.json") + Files.writeString( + file, + """ + { + "format":"Piped","version":1, + "subscriptions":["UC1"], + "groups":[{"groupName":"News","channels":["UC1"]}], + "watchHistory":[{"videoId":"video1","title":"One","watchedAt":12,"currentTime":4}], + "playlists":[{"name":"Saved","videos":["https://youtube.com/watch?v=video1"]}] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = PipedPortabilityAdapter() + + assertEquals(PortabilityFormat.PIPED, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTION_GROUPS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + + val output = ByteArrayOutputStream() + adapter.encode(spool, output, spool.categories()) + assertTrue(output.toString().contains("\"format\":\"Piped\"")) + assertTrue(output.toString().contains("\"groupName\":\"News\"")) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt new file mode 100644 index 00000000..f5963068 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityArchiveInspectorTest.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class PortabilityArchiveInspectorTest { + @TempDir + lateinit var directory: Path + + @Test + fun `inventory reports safe archive entries`() { + val archive = zip("exportInfo" to "{}", "stores/subscriptions" to "[]") + val inventory = requireNotNull(PortabilityArchiveInspector.inspect(archive)) + assertEquals(setOf("exportInfo", "stores/subscriptions"), inventory.names) + assertEquals(4L, inventory.expandedBytes) + } + + @Test + fun `inventory rejects traversal entry names`() { + val archive = zip("../outside" to "invalid") + assertThrows(IllegalArgumentException::class.java) { + PortabilityArchiveInspector.inspect(archive) + } + } + + private fun zip(vararg entries: Pair): Path { + val path = Files.createTempFile(directory, "archive-", ".zip") + ZipOutputStream(Files.newOutputStream(path)).use { output -> + entries.forEach { (name, value) -> + output.putNextEntry(ZipEntry(name)) + output.write(value.toByteArray()) + output.closeEntry() + } + } + return path + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt new file mode 100644 index 00000000..4df0f853 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt @@ -0,0 +1,204 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Path + +class PortabilityEngineTest { + @TempDir + lateinit var directory: Path + + @Test + fun `preview is isolated by owner and can be applied`() = runBlocking { + val dataPort = FakeDataPort() + val engine = engine(dataPort) + val upload = directory.resolve("input.json") + Files.writeString(upload, "fixture") + + val started = engine.startImportPreview("owner-a", upload, "input.json", "application/json") + val ready = awaitState(engine, "owner-a", started.id, PortabilityJobState.READY) + + assertEquals(PortabilityFormat.NEW_PIPE, ready.preview?.detection?.format) + assertEquals(1L, ready.preview?.counts?.get("subscriptions")) + assertThrows(PortabilityJobNotFoundException::class.java) { + engine.snapshot("owner-b", started.id) + } + + engine.applyImport( + "owner-a", + started.id, + PortabilityImportRequest(setOf(PortabilityCategory.SUBSCRIPTIONS)), + ) + val completed = awaitState(engine, "owner-a", started.id, PortabilityJobState.COMPLETED) + assertEquals(1L, completed.result?.get("subscriptions")) + engine.close() + } + + @Test + fun `export produces an owned artifact`() = runBlocking { + val engine = engine(FakeDataPort()) + val started = engine.startExport("owner", PortabilityFormat.NEW_PIPE, setOf(PortabilityCategory.SUBSCRIPTIONS)) + val completed = awaitState(engine, "owner", started.id, PortabilityJobState.COMPLETED) + + assertNotNull(completed.preview) + assertEquals("exported", Files.readString(engine.artifact("owner", started.id))) + engine.close() + } + + @Test + fun `failed job keeps safe diagnostics for support`() = runBlocking { + val engine = engine(FakeDataPort(), FailingAdapter()) + val upload = directory.resolve("invalid.json") + Files.writeString(upload, "fixture") + + val started = engine.startImportPreview( + "owner", + upload, + "invalid.json", + "application/json", + requestId = "request-portability-test", + ) + val failed = awaitState(engine, "owner", started.id, PortabilityJobState.FAILED) + + assertEquals("request-portability-test", failed.requestId) + assertEquals("portability_invalid_input", failed.errorCode) + assertEquals("Unsupported backup version", failed.errorMessage) + assertEquals(failed.requestId, engine.report("owner", started.id).requestId) + engine.close() + } + + @Test + fun `cancelled analysis stops before its files can be deleted`() = runBlocking { + val engine = engine(FakeDataPort(), SlowAdapter()) + val upload = directory.resolve("slow.json") + Files.writeString(upload, "fixture") + val started = engine.startImportPreview("owner", upload, "slow.json", "application/json") + var progressed = false + repeat(100) { + if (!progressed) { + progressed = (engine.snapshot("owner", started.id).progress?.processed ?: 0L) > 0L + if (!progressed) delay(5) + } + } + assertEquals(true, progressed) + + engine.cancel("owner", started.id) + repeat(100) { + if (runCatching { engine.delete("owner", started.id) }.isSuccess) { + assertThrows(PortabilityJobNotFoundException::class.java) { engine.snapshot("owner", started.id) } + engine.close() + return@runBlocking + } + delay(5) + } + error("Cancelled portability analysis did not stop") + } + + private fun engine(dataPort: PortabilityDataPort, adapter: PortabilityAdapter = FakeAdapter()): PortabilityEngine { + val store = PortabilityJobStore(directory.resolve("jobs")) + return PortabilityEngine( + PortabilityRegistry(listOf(adapter)), + dataPort, + store, + CoroutineScope(SupervisorJob() + Dispatchers.Default), + ) + } + + private suspend fun awaitState( + engine: PortabilityEngine, + owner: String, + id: String, + expected: PortabilityJobState, + ): PortabilityJobSnapshot { + repeat(100) { + val snapshot = engine.snapshot(owner, id) + if (snapshot.state == expected) return snapshot + if (snapshot.state == PortabilityJobState.FAILED) error("Portability job failed") + delay(10) + } + error("Portability job did not reach $expected") + } +} + +private class SlowAdapter : PortabilityAdapter by FakeAdapter() { + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + repeat(10_000) { index -> + Thread.sleep(1) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC$index")) + } + } +} + +private class FailingAdapter : PortabilityAdapter by FakeAdapter() { + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + throw IllegalArgumentException("Unsupported backup version") + } +} + +private open class FakeAdapter : PortabilityAdapter { + override val descriptor = PortabilityAdapterDescriptor( + PortabilityFormat.NEW_PIPE, + 1, + setOf( + PortabilityCapability( + PortabilityCategory.SUBSCRIPTIONS, + setOf(PortabilityDirection.IMPORT, PortabilityDirection.EXPORT), + PortabilityFidelity.COMPLETE, + ), + ), + "json", + "application/json", + ) + + override fun detect(input: PortabilityInput) = PortabilityDetection( + PortabilityFormat.NEW_PIPE, + "1", + 100, + "test fixture", + ) + + override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1")) + } + + override fun encode( + source: PortabilityRecordSource, + output: OutputStream, + categories: Set, + ) { + output.write("exported".toByteArray()) + } +} + +private class FakeDataPort : PortabilityDataPort { + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ): Map = source.counts().mapKeys { it.key.wireName }.also { result -> + request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) { + sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) + sink.write(PortabilitySubscription("https://youtube.com/channel/UC1")) + onCategoryComplete(PortabilityCategory.SUBSCRIPTIONS, 1L) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt new file mode 100644 index 00000000..b845d7cc --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt @@ -0,0 +1,39 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityJobStoreTest { + @TempDir + lateinit var directory: Path + + @Test + fun `cleanup never removes active jobs`() { + var now = 1_000L + val store = PortabilityJobStore(directory.resolve("jobs"), { now }, retentionMs = 100L) + val job = store.create("owner", PortabilityJobKind.EXPORT) + + now += 1_000L + store.cleanup() + + assertEquals(job.id, store.get("owner", job.id).id) + store.close() + } + + @Test + fun `cleanup removes completed jobs after retention`() { + var now = 1_000L + val store = PortabilityJobStore(directory.resolve("jobs"), { now }, retentionMs = 100L) + val job = store.create("owner", PortabilityJobKind.EXPORT) + job.transition(setOf(PortabilityJobState.QUEUED), PortabilityJobState.COMPLETED) + + now += 1_000L + store.cleanup() + + assertThrows(PortabilityJobNotFoundException::class.java) { store.get("owner", job.id) } + store.close() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt new file mode 100644 index 00000000..807ce70a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt @@ -0,0 +1,59 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilitySpoolTest { + @TempDir + lateinit var directory: Path + + @Test + fun `spool deduplicates stable records and preserves insertion order`() { + val spool = PortabilitySpool.create(directory) + val first = PortabilitySubscription("https://youtube.com/channel/one", "One") + val duplicate = first.copy(name = "Renamed") + val second = PortabilitySubscription("https://youtube.com/channel/two", "Two") + + assertTrue(spool.write(first).inserted) + assertFalse(spool.write(duplicate).inserted) + assertTrue(spool.write(second).inserted) + + val records = mutableListOf() + spool.forEach(PortabilityCategory.SUBSCRIPTIONS, records::add) + assertEquals(listOf(first, second), records) + assertEquals(mapOf(PortabilityCategory.SUBSCRIPTIONS to 2L), spool.counts()) + assertEquals(1L, spool.duplicateCount()) + spool.delete() + } + + @Test + fun `spool aggregates matching issues`() { + val spool = PortabilitySpool.create(directory) + val issue = PortabilityIssue( + PortabilityCategory.SETTINGS, + "unsupported_setting", + "A source setting cannot be represented", + count = 2, + ) + spool.issue(issue) + spool.issue(issue.copy(count = 3)) + + assertEquals(listOf(issue.copy(count = 5)), spool.issues()) + spool.delete() + } + + @Test + fun `spool preserves present empty categories`() { + val spool = PortabilitySpool.create(directory) + + spool.markCategory(PortabilityCategory.HISTORY) + + assertEquals(mapOf(PortabilityCategory.HISTORY to 0L), spool.counts()) + assertEquals(setOf(PortabilityCategory.HISTORY), spool.categories()) + spool.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt new file mode 100644 index 00000000..104975e6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.portability + +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path + +class TypeTypePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `detects formatted TypeType backups`() { + val file = directory.resolve("formatted-typetype.json") + Files.writeString( + file, + """ + { + "format": "typetype-backup", + "version": 1, + "subscriptions": [] + } + """.trimIndent(), + ) + val input = PortabilityInputFactory.create(file, file.fileName.toString(), "application/json") + + assertEquals(PortabilityFormat.TYPE_TYPE, TypeTypePortabilityAdapter().detect(input)?.format) + } + + @Test + fun `legacy TypeType backup round trips through canonical records`() { + val source = PortabilitySpool.create(directory) + source.write(PortabilitySubscription("https://youtube.com/channel/UC1", "Channel", subscribedAt = 10)) + source.write(PortabilitySubscriptionGroup("News")) + source.write(PortabilitySubscriptionGroupMembership("News", "https://youtube.com/channel/UC1")) + source.write(PortabilityPlaylist("playlist-1", "Saved", "Description", 20)) + source.write( + PortabilityPlaylistVideo( + "playlist-1", + 0, + PortabilityVideo("https://youtube.com/watch?v=video000001", "Video"), + 30, + ), + ) + source.write(PortabilitySettings(buildJsonObject { put("defaultQuality", "720p") })) + source.write(PortabilityContentFilter("blockedKeyword", "spoiler", createdAt = 40)) + + val adapter = TypeTypePortabilityAdapter() + val output = ByteArrayOutputStream() + adapter.encode(source, output, source.categories()) + val file = directory.resolve("typetype.json") + Files.write(file, output.toByteArray()) + val input = PortabilityInputFactory.create(file, "typetype.json", "application/json") + val restored = PortabilitySpool.create(directory) + + assertEquals(PortabilityFormat.TYPE_TYPE, adapter.detect(input)?.format) + adapter.decode(input, restored) + + assertEquals(source.counts(), restored.counts()) + assertTrue(output.toString().contains("\"subscriptionGroups\"")) + assertTrue(output.toString().contains("\"blockedKeywords\"")) + restored.delete() + source.delete() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt new file mode 100644 index 00000000..601fb0eb --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.outputStream + +class ViewTubePortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports viewtube subscriptions history and progress`() { + val archive = directory.resolve("viewtube.zip") + ZipOutputStream(archive.outputStream()).use { zip -> + zip.putNextEntry(ZipEntry("user.json")) + zip.write(VIEW_TUBE_USER.toByteArray()) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = ViewTubePortabilityAdapter() + + assertEquals(PortabilityFormat.VIEW_TUBE, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.PROGRESS]) + var history: PortabilityHistory? = null + spool.forEach(PortabilityCategory.HISTORY) { history = it as PortabilityHistory } + assertEquals("https://www.youtube.com/watch?v=video1", history?.video?.url) + assertEquals("A video", history?.video?.title) + assertEquals(42L, history?.positionSeconds) + assertTrue(spool.issues().any { it.code == "viewtube_settings_ignored" }) + spool.delete() + } + + @Test + fun `does not advertise a viewtube export that viewtube cannot restore`() { + val capability = ViewTubePortabilityAdapter().descriptor.capabilities + assertTrue(capability.isNotEmpty()) + assertTrue(capability.all { it.directions == setOf(PortabilityDirection.IMPORT) }) + assertNotNull(capability.firstOrNull { it.category == PortabilityCategory.HISTORY }) + } +} + +private val VIEW_TUBE_USER = """ + { + "username":"alice", + "subscriptions":{"channels":[{ + "authorId":"UC1","author":"Channel","authorUrl":"https://www.youtube.com/channel/UC1", + "authorThumbnails":[{"url":"avatar","width":88,"height":88}] + }],"channelCount":1}, + "history":{"videos":[{ + "videoId":"video1","progressSeconds":42,"lengthSeconds":120,"lastVisit":"2026-08-20T10:15:30Z", + "videoDetails":{"videoId":"video1","title":"A video","author":"Channel","authorId":"UC1", + "videoThumbnails":[{"url":"thumb","width":480,"height":360}],"viewCount":12,"lengthSeconds":120} + }],"videoCount":1}, + "settings":{"theme":"dark"} + } +""".trimIndent() diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt new file mode 100644 index 00000000..84fb2371 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -0,0 +1,71 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter streams takeout categories and keeps playlist order`() { + val archive = directory.resolve("takeout.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube and YouTube Music/subscriptions/subscriptions.csv", + "Channel Id,Channel Url,Channel Title\nUC123456789012,https://youtube.com/channel/UC123456789012,Channel\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/playlists.csv", + "Playlist ID,Playlist Title\nPL123456789,Imported\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Videos de Imported.csv", + "Video ID,Video Title,Video Added Timestamp\nvideo000001,First,2026-01-02T00:00:00Z\nvideo000002,Second,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Watch later.csv", + "Video ID,Video Title\nwatch000001,Later\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Liked videos.csv", + "Video ID,Video Title\nliked000001,Liked\n", + ) + output.entry( + "Takeout/My Activity/YouTube/watch-history.html", + "You watched Seen
1 Jan 2026, 12:00:00 CET
", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = YoutubeTakeoutPortabilityAdapter() + + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + val positions = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> + positions += (record as PortabilityPlaylistVideo).position + } + assertEquals(listOf(0, 1), positions) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt index dd1e0683..ec12e743 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt @@ -3,12 +3,46 @@ package dev.typetype.server.routes import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.StreamService +import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.testStreamResponse +import io.ktor.client.request.post +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.call +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.post +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test class SabrPlaybackAccessValidatorTest { + @Test + fun `unrestricted playback does not wait for full stream metadata`() = testApplication { + val store = mockk(relaxed = true) + val streams = mockk() + coEvery { store.fetchInfo("video-id", 0L, cachedFirst = true) } returns null + application { + install(ContentNegotiation) { json() } + val handler = SabrPlaybackHandler(store, streams, null, null, null) + routing { + post("/sabr/playback/{videoId}") { + handler.create(call, call.parameters["videoId"].orEmpty()) + } + } + } + + val response = client.post("/sabr/playback/video-id") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + coVerify(exactly = 0) { streams.getStreamInfo(any()) } + } + @Test fun `uses linked YouTube session even when public metadata is accessible`() = runBlocking { val authenticated = ExtractionResult.Success(testStreamResponse().copy(title = "Authenticated")) diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt index e813a8a5..17aeaeac 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt @@ -31,7 +31,7 @@ class SabrPlaybackWindowBuilderTest { val streamState = mockk(relaxed = true) every { session.streamState } returns streamState every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 491_203L) } returns 98 - every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 488_200L) } returns 49 + every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 491_203L) } returns 49 every { session.getCachedSegment(any()) } answers { firstArg().takeIf { it.format.itag == 299 && it.sequenceNumber == 101 } ?.let { mediaSegment(sequence = 101, startMs = 488_200L, durationMs = 6_500L) } diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt new file mode 100644 index 00000000..031d11ce --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -0,0 +1,122 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionStore +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrProgressivePlaybackWindowTest { + @Test + fun `vod audio starts at the playhead rather than the preceding video keyframe`() = runTest { + val audio = format(140, isAudio = true) + val video = format(137, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(video, 100_000L) } returns 16 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 100_000L) } returns 11 + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getReadableSegment(match { it.format == video && it.sequenceNumber == 16 }) } returns + readableSegment(video, sequence = 16, startMs = 95_000L, durationMs = 7_000L) + every { session.getReadableSegment(match { it.format == audio && it.sequenceNumber == 11 }) } returns + readableSegment(audio, sequence = 11, startMs = 99_845L, durationMs = 9_985L) + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest(0L, 100_000L, video.itag, audio.itag, bufferGoalMs = 1_000L), + ) + + assertTrue(result.isReady) + assertEquals(95_000L, requireNotNull(result.response.video).segments.single().startMs) + assertEquals(99_845L, result.response.audio.segments.single().startMs) + } + + @Test + fun `vod window exposes timeline segments before their payload completes`() = runTest { + val audio = format(140, isAudio = true) + val video = format(308, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 1 + every { state.getSegmentStartMs(any(), 1) } returns 0L + every { state.getSegmentEndMs(audio, 1) } returns 9_985L + every { state.getSegmentEndMs(video, 1) } returns 4_000L + val session = mockk(relaxed = true) + val audioSegment = readableSegment(audio, durationMs = 9_985L) + val videoSegment = readableSegment(video, durationMs = 4_000L) + every { session.streamState } returns state + every { session.getReadableSegment(match { it.format == audio }) } returns audioSegment + every { session.getReadableSegment(match { it.format == video }) } returns videoSegment + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest(0L, 0L, video.itag, audio.itag, bufferGoalMs = 2_500L), + ) + + assertTrue(result.isReady) + assertTrue(result.blockedRequests.isEmpty()) + assertFalse(audioSegment.isComplete) + assertFalse(videoSegment.isComplete) + assertEquals(4_000L, requireNotNull(result.response.video).segments.single().durationMs) + assertEquals(9_985L, result.response.audio.segments.single().durationMs) + } + + private fun readableSegment( + format: YoutubeSabrFormat, + sequence: Int = 1, + startMs: Long = 0L, + durationMs: Long, + ): SabrMediaSegment { + val header = mockk() + every { header.itag } returns format.itag + every { header.isInitSegment } returns false + every { header.sequenceNumber } returns sequence + every { header.startMs } returns startMs + every { header.durationMs } returns durationMs + return mockk().also { + every { it.header } returns header + every { it.isComplete } returns false + } + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk().also { + every { it.itag } returns itag + every { it.isAudio } returns isAudio + every { it.mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { it.approxDurationMs } returns 900_000L + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt new file mode 100644 index 00000000..4c6e2828 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt @@ -0,0 +1,84 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Test +import java.time.Duration + +@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) +class AuthenticatedSabrInfoCacheTest { + @Test + fun `concurrent timeout callers share one bounded load`() = runTest { + val cache = AuthenticatedSabrInfoCache(timeoutMs = 100L) + var loads = 0 + val calls = List(3) { + async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + } + runCurrent() + advanceTimeBy(100L) + runCurrent() + + calls.forEach { call -> + assertInstanceOf(TimeoutCancellationException::class.java, runCatching { call.await() }.exceptionOrNull()) + } + assertEquals(1, loads) + } + + @Test + fun `cancelled leader does not poison a following caller`() = runTest { + val cache = AuthenticatedSabrInfoCache(timeoutMs = 100L) + var loads = 0 + val leader = async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + runCurrent() + leader.cancelAndJoin() + + val follower = async { cache.getOrLoad(credentials, VIDEO_ID) { loads++; awaitCancellation() } } + runCurrent() + advanceTimeBy(100L) + runCurrent() + + assertInstanceOf(TimeoutCancellationException::class.java, runCatching { follower.await() }.exceptionOrNull()) + assertEquals(2, loads) + } + + @Test + fun `successful concurrent callers share one ready result`() = runTest { + val cache = AuthenticatedSabrInfoCache(Duration.ofMinutes(1), timeoutMs = 100L) + var loads = 0 + val prepared = mockkPrepared() + val calls = List(3) { + async { + cache.getOrLoad(credentials, VIDEO_ID) { + loads++ + advanceTimeBy(50L) + AuthenticatedSabrInfoResult.Ready(prepared) + } + } + } + advanceTimeBy(50L) + runCurrent() + + calls.forEach { call -> assertEquals(prepared, (call.await() as AuthenticatedSabrInfoResult.Ready).prepared) } + assertEquals(1, loads) + } + + private fun mockkPrepared(): SabrPreparedInfo = mockk() + + private companion object { + val credentials = YoutubeSessionCredentials( + userId = "user", + fingerprint = "fingerprint", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + const val VIDEO_ID = "video-id" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt index 9b6e3449..ffdb5440 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -6,7 +6,7 @@ import io.mockk.every import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertThrows @@ -17,7 +17,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo class AuthenticatedSabrInfoServiceTest { @Test - fun `connected account uses one authenticated token pair`() = runTest { + fun `connected account uses one authenticated token pair`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val probe = mockk() @@ -48,7 +48,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `reuses authenticated info for the following playback request`() = runTest { + fun `reuses authenticated info for the following playback request`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val probe = mockk() @@ -73,7 +73,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `guest playback does not inspect connected credentials`() = runTest { + fun `guest playback does not inspect connected credentials`() = runBlocking { val sessions = mockk() val tokenClient = mockk() val service = AuthenticatedSabrInfoService(sessions, tokenClient) @@ -86,7 +86,7 @@ class AuthenticatedSabrInfoServiceTest { } @Test - fun `authenticated probe failure is typed and does not mark session used`() = runTest { + fun `authenticated probe failure is typed and does not mark session used`() = runBlocking { val sessions = mockk() val tokenClient = mockk() coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) @@ -119,7 +119,7 @@ class AuthenticatedSabrInfoServiceTest { ) assertThrows(CancellationException::class.java) { - runTest { service.fetch(USER_ID, VIDEO_ID) } + runBlocking { service.fetch(USER_ID, VIDEO_ID) } } } diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt new file mode 100644 index 00000000..549950a8 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt @@ -0,0 +1,145 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +class AuthenticatedSabrTimeoutTest { + @Test + fun `visitor data timeout returns a typed failure`() = runBlocking { + val service = service(visitor = { blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `token timeout returns a typed failure`() = runBlocking { + val service = service(token = { _, _, _ -> blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `probe timeout returns a typed failure`() = runBlocking { + val service = service(probe = { _, _ -> blockingCall() }) + + val result = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, result) + } + + @Test + fun `timeout releases credentials for the next request`() = runBlocking { + var calls = 0 + val service = service(visitor = { + calls++ + if (calls == 1) blockingCall() else SESSION_BINDING + }) + + val timedOut = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + val recovered = withTimeout(4_000L) { service.fetch(USER_ID, VIDEO_ID) } + + assertEquals(AuthenticatedSabrInfoResult.TimedOut, timedOut) + assertEquals(2, calls) + assertTrue(recovered is AuthenticatedSabrInfoResult.Ready, recovered.toString()) + } + + @Test + fun `authenticated stream metadata timeout is typed`() = runTest { + val metadata = mockk() + coEvery { metadata.getStreamInfo(USER_ID, URL) } coAnswers { + delay(60_000L) + error("unreachable") + } + val service = YoutubeSessionSabrStreamService( + metadata, + mockk(relaxed = true), + timeoutMs = 20L, + ) + + val result = withTimeout(1_000L) { service.getStreamInfo(USER_ID, URL) } + + assertTrue(result is ExtractionResult.Failure) + assertEquals(AuthenticatedSabrPolicy.TIMEOUT_CODE, (result as ExtractionResult.Failure).code) + } + + private fun service( + visitor: () -> String = { SESSION_BINDING }, + token: (String, String, Boolean) -> SabrTokenBundle? = { _, _, _ -> sessionToken() }, + probe: (String, YoutubeSessionPoToken) -> YoutubeSabrInfo = { _, _ -> playableInfo() }, + ): AuthenticatedSabrInfoService { + val sessions = mockk() + val tokenClient = mockk() + val probeMock = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials() + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(any(), any(), any()) } answers { + token(arg(0), arg(1), arg(2)) + } + every { probeMock.fetch(any(), any()) } answers { probe(arg(0), arg(1)) } + return AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = visitor, + probe = probeMock, + cache = AuthenticatedSabrInfoCache(timeoutMs = 1_200L), + ) + } + + private fun blockingCall(): Nothing { + Thread.sleep(5_000L) + error("unreachable") + } + + private fun playableInfo(): YoutubeSabrInfo = mockk { + every { formats } returns listOf( + mockk { + every { isAudio } returns true + every { isVideo } returns false + }, + mockk { + every { isAudio } returns false + every { isVideo } returns true + }, + ) + } + + private fun credentials() = YoutubeSessionCredentials( + userId = USER_ID, + fingerprint = "fingerprint", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + + private fun sessionToken() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "public-session-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "public-visitor", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + sessionBinding = SESSION_BINDING, + sessionBoundPoToken = "connected-session-token", + ) + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + const val URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val SESSION_BINDING = "connected-visitor" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt index bbcf63f7..9be05aad 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt @@ -17,8 +17,9 @@ class SabrBootstrapStreamServiceTest { val sessionStore = mockk() val tokenClient = mockk() val prepared = preparedInfo() - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns prepared - coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession(prepared.info) + val session = tokenSession(prepared.info, tokenBundle()) + coEvery { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } returns Unit + coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns session val service = SabrBootstrapStreamService(sessionStore, tokenClient) val result = service.getStreamInfo(YOUTUBE_URL) @@ -27,7 +28,8 @@ class SabrBootstrapStreamServiceTest { assertEquals("Bootstrap title", response.title) assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) assertEquals(listOf(140), response.audioStreams.map { it.itag }) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { sessionStore.fetchInfo(any(), any()) } + coVerify(exactly = 1) { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } coVerify(exactly = 1) { tokenClient.fetchPlaybackSession(VIDEO_ID) } } @@ -58,12 +60,22 @@ class SabrBootstrapStreamServiceTest { every { audio.audioTrackId } returns "en.4" val info = mockk() every { info.formats } returns listOf(video, audio) + every { info.visitorData } returns VISITOR_DATA return SabrPreparedInfo(info, null) } - private fun tokenSession(info: YoutubeSabrInfo): TokenYoutubeSession = TokenYoutubeSession( + private fun tokenBundle() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "player-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = VISITOR_DATA, + videoBoundPoToken = "media-token", + videoBoundPoTokenBytes = byteArrayOf(2), + ) + + private fun tokenSession(info: YoutubeSabrInfo, token: SabrTokenBundle? = null): TokenYoutubeSession = TokenYoutubeSession( info = info, - token = null, + token = token, title = "Bootstrap title", author = "Bootstrap channel", channelId = "channel-id", @@ -80,5 +92,6 @@ class SabrBootstrapStreamServiceTest { private companion object { const val VIDEO_ID = "f6f3PhauXyg" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val VISITOR_DATA = "visitor-data" } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt index 6607bafb..facd8627 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt @@ -88,7 +88,7 @@ class SabrDemandWatchdogBackoffTest { } runCurrent() - advanceTimeBy(45_900L) + advanceTimeBy(44_900L) runCurrent() assertFalse(job.isCompleted) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt index 8bf94800..59b34411 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt @@ -3,7 +3,6 @@ package dev.typetype.server.services import dev.typetype.server.routes.SabrPlaybackRecovery import io.mockk.coEvery import io.mockk.coVerify -import io.mockk.coVerifyOrder import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -49,6 +48,10 @@ class SabrPlaybackInitializationFailureTest { delay(Long.MAX_VALUE) null } + coEvery { store.fetchInitializationData(holder, audio) } coAnswers { + delay(Long.MAX_VALUE) + null + } coEvery { store.invalidatePlaybackInfo("video") } returns Unit val result = SabrPlaybackSessionService(store).prepare( @@ -65,7 +68,7 @@ class SabrPlaybackInitializationFailureTest { assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) verify(exactly = 0) { store.startPump(any()) } - coVerify(exactly = 0) { store.fetchInitializationData(holder, audio) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } @Test @@ -113,10 +116,8 @@ class SabrPlaybackInitializationFailureTest { assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) verify(exactly = 0) { store.startPump(any()) } coVerify(exactly = 1) { store.invalidatePlaybackInfo("video") } - coVerifyOrder { - store.fetchInitializationData(holder, video) - store.fetchInitializationData(holder, audio) - } + coVerify(exactly = 1) { store.fetchInitializationData(holder, video) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } @Test diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt index af76683c..420f27b0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt @@ -7,7 +7,6 @@ import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertSame @@ -107,19 +106,19 @@ class SabrPlaybackSessionServiceTest { } @Test - fun `cached media segment after demand returns bytes`() = runTest { + fun `readable media segment after demand returns progressive stream`() = runTest { val audio = format(140, isAudio = true) val holder = holder(audio, format(137, isAudio = false)) - val segment = cachedSegment(140, 4, byteArrayOf(1, 2, 3)) + val segment = mockk() + every { holder.session.awaitReadableSegment(any(), any()) } returns segment val store = mockk() - coEvery { store.cachedSegment(holder, any()) } returnsMany listOf(null, segment) every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit val result = SabrPlaybackSessionService(store).fetchMedia(holder, audio, sequence = 4, timeoutMs = 50L, generation = 0L) - val ready = result as SabrPlaybackSegmentResult.Ready - assertEquals("audio/mp4", ready.mimeType) - assertArrayEquals(byteArrayOf(1, 2, 3), ready.bytes) + val stream = result as SabrPlaybackSegmentResult.Stream + assertEquals("audio/mp4", stream.mimeType) + assertSame(segment, stream.segment) verify(exactly = 1) { store.requestSegmentDemand(holder, any(), 0L) } } @@ -245,6 +244,8 @@ class SabrPlaybackSessionServiceTest { val state = mockk(relaxed = true) every { session.streamState } returns state every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null + every { session.awaitReadableSegment(any(), any()) } returns null every { session.isBeyondEnd(any()) } returns false every { session.prepareForInitialization(any()) } returns Unit every { state.setActiveTrackTypes(any(), any()) } returns Unit diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index f07ad191..25c24a4c 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -170,7 +170,7 @@ class SabrSeekRepositionPumpTest { SabrSessionPumpLoop().run({ rounds++ < 1 }, holder, intervalMs = 0L) verify(exactly = 1) { session.prepareForRewind(request) } - verify(exactly = 1) { session.prepareForMediaSegment(request) } + verify(exactly = 0) { session.prepareForMediaSegment(request) } verify(exactly = 1) { state.setBufferedRangesOverride(null) } verify(exactly = 1) { session.pumpOnceStreamingForDemand(any(), request) } } finally { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt index 2be01656..4552ef57 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt @@ -2,9 +2,10 @@ package dev.typetype.server.services import io.mockk.every import io.mockk.mockk +import io.mockk.verify import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment @@ -17,27 +18,19 @@ import java.time.Instant class SabrSegmentCacheTest { @Test - fun `cache stores and reads segment metadata and bytes`() { + fun `vod cache observes media without copying segment bytes`() { val segmentCache = SabrSegmentCache() val audio = format(140, isAudio = true) val video = format(137, isAudio = false) - val holder = holder(audio, video) + val holder = holder(audio, video, SabrSessionPurpose.PLAYBACK) val bytes = byteArrayOf(1, 2, 3) val segment = segment(itag = 140, sequence = 4, bytes = bytes) val request = SabrSegmentRequest.media(audio, 4) segmentCache.put(holder, segment) - val cached = segmentCache.get(holder, request) - - requireNotNull(cached) - assertEquals(140, cached.itag) - assertEquals(4, cached.sequence) - assertEquals(1234L, cached.startMs) - assertEquals(9985L, cached.durationMs) - assertEquals("audio/mp4", cached.mimeType) - assertArrayEquals(byteArrayOf(1, 2, 3), cached.bytes) - assertSame(bytes, cached.bytes) + assertNull(segmentCache.get(holder, request)) assertEquals(4, holder.observedMediaSegment(audio)?.header?.sequenceNumber) + verify(exactly = 0) { segment.data } } @Test @@ -56,7 +49,11 @@ class SabrSegmentCacheTest { assertArrayEquals(media, requireNotNull(segmentCache.get(holder, request)).bytes) } - private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): SabrSessionHolder { + private fun holder( + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + purpose: SabrSessionPurpose = SabrSessionPurpose.MANIFEST, + ): SabrSessionHolder { val session = mockk() val state = mockk() every { session.streamState } returns state @@ -67,7 +64,7 @@ class SabrSegmentCacheTest { audioFormat = audio, videoFormat = video, sessionToken = "token", - key = SabrSessionKey("video", "user", 140, null, 137, 0L), + key = SabrSessionKey("video", "user", 140, null, 137, 0L, purpose), lastRequestAt = Instant.EPOCH, ) }