From 9b6ca0d93f93920f304fee22656f0afb0e68187a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 11 May 2026 21:41:37 +0200 Subject: [PATCH 01/17] docs: align README with TypeType style --- README.md | 142 +++++++++++++++++++++++++++++------------------------- 1 file changed, 76 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 6f75553..238f009 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,119 @@ -# TypeType-Server +
+

TypeType-Server

+

Extraction and private user data backend for TypeType.

+
-The extraction and user data backend for [TypeType](https://github.com/Priveetee/TypeType). +
-A Kotlin/Ktor server with two responsibilities: -- Wrap [PipePipeExtractor](https://github.com/InfinityLoop1308/PipePipeExtractor) to expose YouTube, NicoNico, and BiliBili extraction as a REST/JSON API -- Persist user data (history, favorites, subscriptions, playlists, watch-later, progress, settings, search history, blocked content) in PostgreSQL +[![Backend](https://img.shields.io/badge/backend-Ktor-7f52ff)](https://ktor.io) +[![Language](https://img.shields.io/badge/language-Kotlin-7f52ff)](https://kotlinlang.org) +[![License](https://img.shields.io/badge/license-GPL%20v3-blue)](LICENSE) -See [Architecture.md](./Architecture.md) for current architecture and API surface overview. +
+ +TypeType-Server is the HTTP API behind TypeType. It wraps PipePipeExtractor for media extraction and stores private user data in PostgreSQL. The frontend talks to this server over HTTP only. + +## What this is + +A Kotlin/Ktor backend for extraction, user data, and downloader gateway routes. + +It provides stream metadata, search, trending, comments, channels, user libraries, settings, import flows, and `/downloader/*` proxying for the native downloader service. + +## What this is not + +- Not a frontend source tree. +- Not a standalone YouTube clone. +- Not a downloader. Download jobs are handled by TypeType-Downloader. +- Not affiliated with YouTube, NicoNico, BiliBili, Piped, Invidious, or NewPipe. ## Stack | Role | Tool | |---|---| | Language | Kotlin | -| Server | Ktor (Netty engine) | -| Extraction | PipePipeExtractor (Java, GPL v3) | -| Build | Gradle (Kotlin DSL) | +| Server | Ktor with Netty | +| Extraction | PipePipeExtractor | +| Build | Gradle Kotlin DSL | | User data | PostgreSQL via Exposed + HikariCP | -| Cache | Dragonfly (Redis-compatible) | +| Cache | Dragonfly | +| Downloader gateway | TypeType-Downloader over HTTP | +| Token service | TypeType-Token over HTTP | -## Development +## Services -### Prerequisites +| Area | Purpose | +|---|---| +| Extraction | Streams, manifests, search, suggestions, trending, comments, channels | +| User data | History, subscriptions, playlists, favorites, watch later, progress, settings | +| Imports | YouTube Takeout and PipePipe backup ingestion | +| Proxying | Media proxy, storyboard proxy, NicoNico video proxy, downloader gateway | +| Admin | Instance metadata, users, sessions, bug reports | -- JDK 17+ -- Docker and Docker Compose (for PostgreSQL and Dragonfly) +## Development -### Start dependencies +Start local dependencies: ```bash cp .env.example .env docker compose up -d postgres dragonfly ``` -### Start full dev mirror stack (frontend + server + downloader) +Build the server jar: ```bash -docker compose -f docker-compose.dev-mirror.yml up -d -./scripts/bootstrap-garage.sh +./gradlew shadowJar ``` -Stack endpoints: - -- Frontend: `http://localhost:28082` -- API server: `http://localhost:28080` -- Downloader: `http://localhost:28093` -- Token service: `http://localhost:28081` - -Pull latest beta images before restart: +Run it locally: ```bash -./scripts/pull-dev-images.sh -docker compose -f docker-compose.dev-mirror.yml up -d --force-recreate +java -jar build/libs/typetype-server-all.jar ``` -### Build +The server listens on `http://localhost:8080`. -```bash -./gradlew shadowJar -``` +## Dev mirror stack -### Run +Run the frontend, backend, downloader, token service, database, cache, and Garage mirror stack: ```bash -java -jar build/libs/typetype-server-all.jar +docker compose -f docker-compose.dev-mirror.yml up -d +./scripts/bootstrap-garage.sh ``` -The server starts on port `8080`. Logs go to stdout. - -### Docker image tags (GHCR) - -Container tags are published to GHCR with: - -- stable image `${{ github.repository }}` on `main` and Git tags `v*` -- beta image `${{ github.repository }}-beta` on `dev` -- `sha-` on every build -- branch tags (`main` on stable image, `dev` on beta image) -- `latest` on default branch (stable image) and on `dev` (beta image) -- `beta` on `dev` (beta image) -- release tags when pushing Git tags like `v1.2.3` (`1.2.3` and `1.2`) on stable image +| Service | URL | +|---|---| +| Frontend | `http://localhost:28082` | +| API server | `http://localhost:28080` | +| Downloader | `http://localhost:28093` | +| Token service | `http://localhost:28081` | -### Configuration +## Configuration -All configuration is via environment variables. Defaults in `.env.example` work for local development. +| Variable | Purpose | +|---|---| +| `ALLOWED_ORIGINS` | Required CORS origins | +| `DATABASE_URL` | PostgreSQL JDBC URL | +| `DATABASE_USER` | PostgreSQL user | +| `DATABASE_PASSWORD` | PostgreSQL password | +| `DRAGONFLY_URL` | Dragonfly connection URL | +| `DOWNLOADER_SERVICE_URL` | Base URL for TypeType-Downloader | +| `SUBTITLE_SERVICE_URL` | Base URL for TypeType-Token | -| Variable | Default | Description | -|---|---|---| -| `ALLOWED_ORIGINS` | — | Comma-separated CORS origins. **Required** — server refuses to start without it. | -| `DATABASE_URL` | `jdbc:postgresql://localhost:5432/typetype` | PostgreSQL JDBC URL | -| `DATABASE_USER` | `typetype` | PostgreSQL user | -| `DATABASE_PASSWORD` | `typetype` | PostgreSQL password | -| `DRAGONFLY_URL` | `redis://localhost:6379` | Dragonfly/Redis URL | -| `DOWNLOADER_SERVICE_URL` | `http://typetype-downloader:18093` | Downloader backend base URL used by `/downloader/*` gateway | +## Checks -## Acknowledgments +```bash +./gradlew test +./gradlew shadowJar +``` -A huge thanks to the projects that made this possible. TypeType-Server is a wrapper, and none of it would exist without the work these teams put in first. +## Related projects -- [InfinityLoop1308/PipePipeExtractor](https://github.com/InfinityLoop1308/PipePipeExtractor) — the extraction engine at the core of this server -- [InfinityLoop1308/PipePipeClient](https://github.com/InfinityLoop1308/PipePipeClient) — reference for consuming PipePipeExtractor in Java -- [InfinityLoop1308/PipePipe](https://github.com/InfinityLoop1308/PipePipe) — reference for multi-service support -- [A-EDev/Flow](https://github.com/A-EDev/Flow) — inspiration for discovery-first recommendation direction and iteration patterns -- [TeamPiped/Piped](https://github.com/TeamPiped/Piped) — API patterns and architecture reference -- [deniscerri/ytdlnis](https://github.com/deniscerri/ytdlnis) — groundbreaking work on YouTube PO token integration that directly shaped the design of TypeType-Token +- [TypeType](https://github.com/Priveetee/TypeType) is the deployment stack. +- [TypeType web](https://github.com/Priveetee/TypeType) is the React frontend. +- [TypeType-Downloader](https://github.com/Priveetee/TypeType-Downloader) handles download artifacts. +- [TypeType-Token](https://github.com/Priveetee/TypeType-Token) provides YouTube PO tokens. ## License -GPL v3 — required by PipePipeExtractor. The TypeType frontend is a separate program communicating over HTTP and is not subject to this license. +GPL v3. This license is required by PipePipeExtractor. From 26b628ab03046c7d50bd22eb526e86cfab3b432d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 11 May 2026 21:48:57 +0200 Subject: [PATCH 02/17] docs: add TypeType banner to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 238f009..973badf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@
+ TypeType

TypeType-Server

Extraction and private user data backend for TypeType.

From f0c2e390f11ae409cf1eb1ef5bb66347ad3a99c9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 13 May 2026 21:27:26 +0200 Subject: [PATCH 03/17] feat: add request observability --- .../kotlin/dev/typetype/server/AppMetrics.kt | 53 +++++++++++++++++ .../kotlin/dev/typetype/server/Plugins.kt | 19 ++---- .../typetype/server/RequestObservability.kt | 58 +++++++++++++++++++ .../typetype/server/models/ErrorResponse.kt | 7 ++- .../services/DownloaderGatewayService.kt | 9 +++ .../server/services/YouTubeSubtitleService.kt | 3 + .../server/DownloaderGatewayRequestIdTest.kt | 43 ++++++++++++++ .../dev/typetype/server/ProfileRoutesTest.kt | 2 +- .../server/RequestObservabilityTest.kt | 58 +++++++++++++++++++ 9 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/AppMetrics.kt create mode 100644 src/main/kotlin/dev/typetype/server/RequestObservability.kt create mode 100644 src/test/kotlin/dev/typetype/server/DownloaderGatewayRequestIdTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/src/main/kotlin/dev/typetype/server/AppMetrics.kt new file mode 100644 index 0000000..617bfad --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -0,0 +1,53 @@ +package dev.typetype.server + +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.path +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +object AppMetrics { + private val totalRequests = AtomicLong() + private val totalDurationMs = AtomicLong() + private val statusCounts = ConcurrentHashMap() + private val routeCounts = ConcurrentHashMap() + + fun record(call: ApplicationCall) { + val status = call.response.status()?.value ?: 0 + val route = metricPath(call.request.path()) + totalRequests.incrementAndGet() + totalDurationMs.addAndGet(call.requestDurationMs()) + statusCounts.getOrPut(status) { AtomicLong() }.incrementAndGet() + routeCounts.getOrPut("$route|$status") { AtomicLong() }.incrementAndGet() + } + + fun snapshot(): String { + val total = totalRequests.get() + val average = if (total == 0L) 0 else totalDurationMs.get() / total + return buildString { + appendLine("requests.total=$total") + appendLine("requests.durationMs.avg=$average") + statusCounts.toSortedMap().forEach { (status, count) -> + appendLine("requests.status.$status=${count.get()}") + } + routeCounts.toSortedMap().forEach { (key, count) -> + val (route, status) = key.split('|', limit = 2) + appendLine("requests.route.${route.metricKey()}.status.$status=${count.get()}") + } + } + } + +} + +fun metricPath(path: String): String = when { + path.startsWith("/downloader/jobs/") && path.endsWith("/events") -> "/downloader/jobs/{id}/events" + path.startsWith("/downloader/jobs/") && path.endsWith("/artifact") -> "/downloader/jobs/{id}/artifact" + path.startsWith("/downloader/jobs/") && path.endsWith("/cancel") -> "/downloader/jobs/{id}/cancel" + path.startsWith("/downloader/jobs/") -> "/downloader/jobs/{id}" + path.startsWith("/progress/") -> "/progress/{videoUrl}" + path.startsWith("/favorites/") -> "/favorites/{videoUrl}" + path.startsWith("/watch-later/") -> "/watch-later/{videoUrl}" + path.startsWith("/subscriptions/") -> "/subscriptions/{channelUrl}" + else -> path.ifBlank { "/" } +} + +private fun String.metricKey(): String = trim('/').ifBlank { "root" }.replace('/', '.') diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index 2c97d3b..ee8a59d 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -14,9 +14,7 @@ import io.ktor.server.plugins.cors.routing.CORS import io.ktor.server.plugins.ratelimit.RateLimit import io.ktor.server.plugins.ratelimit.RateLimitName import io.ktor.server.plugins.statuspages.StatusPages -import io.ktor.server.request.httpMethod import io.ktor.server.request.path -import io.ktor.server.request.uri import io.ktor.server.response.respond import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory @@ -39,14 +37,9 @@ val USER_DATA_ZONE = RateLimitName("user-data") fun Application.configurePlugins(authService: AuthService) { val log = LoggerFactory.getLogger("RequestLogger") + installRequestObservability() install(CallLogging) { - format { call -> - val method = call.request.httpMethod.value - val path = call.request.path() - val status = call.response.status()?.value ?: 0 - val displayPath = if (path.startsWith("/proxy")) "$path?url=" else call.request.uri - "$method $displayPath -> $status" - } + format(::requestLogLine) } install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) @@ -95,17 +88,17 @@ fun Application.configurePlugins(authService: AuthService) { install(StatusPages) { status(HttpStatusCode.TooManyRequests) { call, status -> if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") - call.respond(status, ErrorResponse("Too many requests")) + call.respond(status, ErrorResponse("Too many requests", "rate_limited")) } exception { call, cause -> log.warn("Bad request: ${cause.message}") - call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request")) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request", "bad_request")) } exception { call, cause -> if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception if (cause is kotlinx.coroutines.CancellationException) throw cause - log.error("Unhandled exception on ${call.request.path()}", cause) - call.respond(HttpStatusCode.InternalServerError, ErrorResponse(cause.message ?: "Internal server error")) + log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) + call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) } } } diff --git a/src/main/kotlin/dev/typetype/server/RequestObservability.kt b/src/main/kotlin/dev/typetype/server/RequestObservability.kt new file mode 100644 index 0000000..a14c6db --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/RequestObservability.kt @@ -0,0 +1,58 @@ +package dev.typetype.server + +import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.ApplicationCallPipeline +import io.ktor.server.request.httpMethod +import io.ktor.server.request.path +import io.ktor.util.AttributeKey +import kotlinx.coroutines.asContextElement +import kotlinx.coroutines.withContext +import java.util.UUID + +const val REQUEST_ID_HEADER = "X-Request-ID" + +private val requestIdAttribute = AttributeKey("requestId") +private val requestStartNanosAttribute = AttributeKey("requestStartNanos") +private val requestIdContext = ThreadLocal() +private val requestIdRegex = Regex("^[A-Za-z0-9._-]{8,128}$") + +fun currentRequestId(): String? = requestIdContext.get() + +fun ApplicationCall.requestId(): String = attributeOrNull(requestIdAttribute) ?: currentRequestId() ?: "unknown" + +fun ApplicationCall.requestDurationMs(): Long { + val startedAt = attributeOrNull(requestStartNanosAttribute) ?: return 0 + return (System.nanoTime() - startedAt).coerceAtLeast(0) / 1_000_000 +} + +fun Application.installRequestObservability() { + intercept(ApplicationCallPipeline.Setup) { + val applicationCall = context + val requestId = resolveRequestId(applicationCall.request.headers[REQUEST_ID_HEADER]) + applicationCall.attributes.put(requestIdAttribute, requestId) + applicationCall.attributes.put(requestStartNanosAttribute, System.nanoTime()) + applicationCall.response.headers.append(REQUEST_ID_HEADER, requestId, safeOnly = false) + withContext(requestIdContext.asContextElement(requestId)) { + try { + proceed() + } finally { + AppMetrics.record(applicationCall) + } + } + } +} + +fun requestLogLine(call: ApplicationCall): String = listOf( + "requestId=${call.requestId()}", + "method=${call.request.httpMethod.value}", + "path=${metricPath(call.request.path())}", + "status=${call.response.status()?.value ?: 0}", + "durationMs=${call.requestDurationMs()}", +).joinToString(" ") + +private fun resolveRequestId(raw: String?): String = + raw?.takeIf { requestIdRegex.matches(it) } ?: UUID.randomUUID().toString() + +private fun ApplicationCall.attributeOrNull(key: AttributeKey): T? = + if (attributes.contains(key)) attributes[key] else null diff --git a/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt b/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt index 74536f3..b1e1950 100644 --- a/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt @@ -1,6 +1,11 @@ package dev.typetype.server.models +import dev.typetype.server.currentRequestId import kotlinx.serialization.Serializable @Serializable -data class ErrorResponse(val error: String) +data class ErrorResponse( + val error: String, + val code: String = "error", + val requestId: String? = currentRequestId(), +) diff --git a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt index a2926ac..d9962ee 100644 --- a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt +++ b/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt @@ -1,5 +1,7 @@ package dev.typetype.server.services +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.currentRequestId import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.OkHttpClient import okhttp3.Request @@ -40,6 +42,7 @@ class DownloaderGatewayService( val requestBuilder = Request.Builder().url(url).method(method, requestBody) headers.forEach { (name, value) -> if (shouldForwardRequestHeader(name)) requestBuilder.addHeader(name, value) } + currentRequestId()?.let { requestBuilder.header(REQUEST_ID_HEADER, it) } return client.newCall(requestBuilder.build()).execute() } @@ -50,6 +53,12 @@ class DownloaderGatewayService( return client.newCall(requestBuilder.build()).execute() } + suspend fun healthCheck(): Boolean = kotlinx.coroutines.withContext(kotlinx.coroutines.Dispatchers.IO) { + client.newCall(Request.Builder().url(buildUrl("/health", null)).get().build()) + .execute() + .use { it.isSuccessful } + } + private fun buildUrl(path: String, query: String?): String { val cleanPath = if (path.startsWith('/')) path else "/$path" val withPath = "${baseUrl.trimEnd('/')}$cleanPath" diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt index ed606f1..18bbdf6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt @@ -1,6 +1,8 @@ package dev.typetype.server.services +import dev.typetype.server.REQUEST_ID_HEADER import dev.typetype.server.cache.CacheJson +import dev.typetype.server.currentRequestId import dev.typetype.server.models.SubtitleItem import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -15,6 +17,7 @@ internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, priv val response = httpClient.newCall( Request.Builder() .url("$baseUrl/subtitles?videoId=$videoId") + .apply { currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } } .build() ).execute() response.use { CacheJson.decodeFromString>(it.body.string()) } diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayRequestIdTest.kt b/src/test/kotlin/dev/typetype/server/DownloaderGatewayRequestIdTest.kt new file mode 100644 index 0000000..d186f15 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/DownloaderGatewayRequestIdTest.kt @@ -0,0 +1,43 @@ +package dev.typetype.server + +import com.sun.net.httpserver.HttpServer +import dev.typetype.server.routes.downloaderGatewayRoutes +import dev.typetype.server.services.DownloaderGatewayService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.http.HttpStatusCode +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.net.InetSocketAddress +import java.util.concurrent.atomic.AtomicReference + +class DownloaderGatewayRequestIdTest { + @Test + fun `downloader gateway forwards request id`() = testApplication { + val forwardedRequestId = AtomicReference() + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/health") { exchange -> + forwardedRequestId.set(exchange.requestHeaders.getFirst(REQUEST_ID_HEADER)) + val payload = "{}".toByteArray() + exchange.sendResponseHeaders(200, payload.size.toLong()) + exchange.responseBody.use { it.write(payload) } + } + upstream.start() + + val gateway = DownloaderGatewayService(baseUrl = "http://127.0.0.1:${upstream.address.port}") + application { + installRequestObservability() + routing { downloaderGatewayRoutes(gateway) } + } + + try { + val response = client.get("/downloader/health") { header(REQUEST_ID_HEADER, "request-test-123") } + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("request-test-123", forwardedRequestId.get()) + } finally { + upstream.stop(0) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt index 9cf16c0..7df873d 100644 --- a/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt @@ -89,7 +89,7 @@ class ProfileRoutesTest { setBody("""{"imageUrl":"https://cdn.test/avatar.gif"}""") } assertEquals(HttpStatusCode.Gone, response.status) - assertEquals("{\"error\":\"AVATAR_MODE_EMOJI_ONLY\"}", response.bodyAsText()) + assertEquals("{\"error\":\"AVATAR_MODE_EMOJI_ONLY\",\"code\":\"error\",\"requestId\":null}", response.bodyAsText()) } @Test diff --git a/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt b/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt new file mode 100644 index 0000000..7d6fe1e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt @@ -0,0 +1,58 @@ +package dev.typetype.server + +import dev.typetype.server.models.ErrorResponse +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.call +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.response.respond +import io.ktor.server.routing.get +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RequestObservabilityTest { + @Test + fun `request id is echoed and included in error body`() = testApplication { + application { + installRequestObservability() + install(ContentNegotiation) { json(Json { encodeDefaults = true }) } + routing { + get("/bad") { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request", "bad_request")) + } + } + } + + val response = client.get("/bad") { header(REQUEST_ID_HEADER, "request-test-123") } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertEquals("request-test-123", response.headers[REQUEST_ID_HEADER]) + val body = response.bodyAsText() + assertTrue(body.contains("\"code\":\"bad_request\"")) + assertTrue(body.contains("\"requestId\":\"request-test-123\"")) + } + + @Test + fun `invalid request id is replaced`() = testApplication { + application { + installRequestObservability() + routing { get("/ok") { call.respond(HttpStatusCode.NoContent) } } + } + + val response = client.get("/ok") { header(REQUEST_ID_HEADER, "bad") } + + assertEquals(HttpStatusCode.NoContent, response.status) + assertNotNull(response.headers[REQUEST_ID_HEADER]) + assertNotEquals("bad", response.headers[REQUEST_ID_HEADER]) + } +} From 60969ca446db035c9e54ddacf30b1d1de282aa2f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 13 May 2026 21:27:39 +0200 Subject: [PATCH 04/17] feat: add internal observability endpoints --- .../kotlin/dev/typetype/server/Application.kt | 4 ++ .../typetype/server/cache/DragonflyService.kt | 2 + .../dev/typetype/server/db/DatabaseFactory.kt | 4 ++ .../server/models/DeepHealthResponse.kt | 9 +++ .../routes/InternalObservabilityRoutes.kt | 37 ++++++++++++ .../server/services/InternalHealthService.kt | 36 +++++++++++ .../server/InternalObservabilityRoutesTest.kt | 59 +++++++++++++++++++ 7 files changed, 151 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt create mode 100644 src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index dbd8d7e..488a34b 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -8,6 +8,7 @@ import dev.typetype.server.routes.bulletCommentRoutes import dev.typetype.server.routes.channelRoutes import dev.typetype.server.routes.commentRoutes import dev.typetype.server.routes.downloaderGatewayRoutes +import dev.typetype.server.routes.internalObservabilityRoutes import dev.typetype.server.routes.manifestRoutes import dev.typetype.server.routes.nicoVideoProxyRoutes import dev.typetype.server.routes.proxyRoutes @@ -34,6 +35,7 @@ import dev.typetype.server.services.ProfileService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.InstanceService +import dev.typetype.server.services.InternalHealthService import dev.typetype.server.services.UserAdminService import io.ktor.server.application.Application import io.ktor.server.netty.EngineMain @@ -72,10 +74,12 @@ fun Application.module() { val svc = ServiceRegistry(cache, subtitleServiceUrl) val downloaderGatewayService = DownloaderGatewayService(downloaderServiceUrl) val openMojiProxyService = OpenMojiProxyService(cache) + val internalHealthService = InternalHealthService(cache, downloaderGatewayService, subtitleServiceUrl) configurePlugins(authService) routing { + internalObservabilityRoutes(internalHealthService::check) publicMetadataRoutes(instanceService::getInstance) rateLimit(STREAMS_ZONE) { streamRoutes(svc.streamService) diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt index ba978a6..32aa51b 100644 --- a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt @@ -19,4 +19,6 @@ class DragonflyService(url: String) : CacheService { override suspend fun delete(key: String): Unit = async.del(key).await().let {} + + suspend fun ping(): Boolean = async.ping().await() == "PONG" } diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 7dbe4e0..fe58b00 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -104,4 +104,8 @@ object DatabaseFactory { suspend fun query(block: () -> T): T = withContext(Dispatchers.IO) { transaction { block() } } + + fun healthCheck(): Boolean = runCatching { + transaction { exec("SELECT 1") { it.next() } == true } + }.getOrDefault(false) } diff --git a/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt b/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt new file mode 100644 index 0000000..c96c87f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class DeepHealthResponse( + val status: String, + val checks: Map, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt new file mode 100644 index 0000000..12fb2f1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt @@ -0,0 +1,37 @@ +package dev.typetype.server.routes + +import dev.typetype.server.AppMetrics +import dev.typetype.server.models.DeepHealthResponse +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.call +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.route + +fun Route.internalObservabilityRoutes( + healthProvider: suspend () -> DeepHealthResponse, + metricsProvider: () -> String = AppMetrics::snapshot, + tokenProvider: () -> String? = { System.getenv("INTERNAL_OBSERVABILITY_TOKEN") }, +) { + route("/internal") { + get("/health/deep") { + if (!call.isInternalAuthorized(tokenProvider)) return@get call.respond(HttpStatusCode.NotFound) + call.respond(healthProvider()) + } + get("/metrics") { + if (!call.isInternalAuthorized(tokenProvider)) return@get call.respond(HttpStatusCode.NotFound) + call.respondText(metricsProvider(), ContentType.Text.Plain) + } + } +} + +private fun ApplicationCall.isInternalAuthorized(tokenProvider: () -> String?): Boolean { + val expected = tokenProvider()?.takeIf { it.isNotBlank() } ?: return false + val provided = request.headers["X-Internal-Token"] ?: request.headers[HttpHeaders.Authorization]?.removePrefix("Bearer ") + return provided == expected +} diff --git a/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt b/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt new file mode 100644 index 0000000..7a5e7bf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.DragonflyService +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.models.DeepHealthResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request + +class InternalHealthService( + private val cache: DragonflyService, + private val downloaderGatewayService: DownloaderGatewayService, + private val tokenServiceUrl: String, + private val client: OkHttpClient = OkHttpClient(), +) { + suspend fun check(): DeepHealthResponse { + val checks = linkedMapOf( + "postgres" to check(DatabaseFactory::healthCheck), + "dragonfly" to check { cache.ping() }, + "downloader" to check(downloaderGatewayService::healthCheck), + "token" to check(::tokenHealthCheck), + ) + val status = if (checks.values.all { it == "ok" }) "ok" else "degraded" + return DeepHealthResponse(status = status, checks = checks) + } + + private suspend fun check(block: suspend () -> Boolean): String = + if (runCatching { block() }.getOrDefault(false)) "ok" else "error" + + private suspend fun tokenHealthCheck(): Boolean = withContext(Dispatchers.IO) { + client.newCall(Request.Builder().url("${tokenServiceUrl.trimEnd('/')}/health").get().build()) + .execute() + .use { it.isSuccessful } + } +} diff --git a/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt new file mode 100644 index 0000000..44b2e85 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt @@ -0,0 +1,59 @@ +package dev.typetype.server + +import dev.typetype.server.models.DeepHealthResponse +import dev.typetype.server.routes.internalObservabilityRoutes +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class InternalObservabilityRoutesTest { + @Test + fun `internal health requires token`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { testRoutes() } + } + + val response = client.get("/internal/health/deep") + + assertEquals(HttpStatusCode.NotFound, response.status) + } + + @Test + fun `internal health returns checks with token`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { testRoutes() } + } + + val response = client.get("/internal/health/deep") { header("X-Internal-Token", "secret") } + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("""{"status":"ok","checks":{"postgres":"ok"}}""", response.bodyAsText()) + } + + @Test + fun `internal metrics returns key value text with token`() = testApplication { + application { routing { testRoutes() } } + + val response = client.get("/internal/metrics") { header("X-Internal-Token", "secret") } + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().contains("requests.total=1")) + } + + private fun io.ktor.server.routing.Route.testRoutes() = internalObservabilityRoutes( + healthProvider = { DeepHealthResponse("ok", mapOf("postgres" to "ok")) }, + metricsProvider = { "requests.total=1" }, + tokenProvider = { "secret" }, + ) +} From 80f074de997eea5b0e389378cba2a181e5464167 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 13 May 2026 21:48:45 +0200 Subject: [PATCH 05/17] chore: update extractor and test dependencies --- build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 78ad288..1e35339 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -29,14 +29,14 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.5.32") - implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:810a08db8efc8bf218e92019c09a83229c6abd1b") + implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:b7109238ab38e16272ea17d925d41a6292bce014") implementation("com.squareup.okhttp3:okhttp:5.3.2") implementation("io.lettuce:lettuce-core:7.5.1.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.2.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.2.0") implementation("com.zaxxer:HikariCP:7.0.2") implementation("org.postgresql:postgresql:42.7.11") - implementation("org.xerial:sqlite-jdbc:3.53.0.0") + implementation("org.xerial:sqlite-jdbc:3.53.1.0") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.5.2") testImplementation("org.junit.jupiter:junit-jupiter:6.0.3") @@ -76,7 +76,7 @@ tasks.test { } jacoco { - toolVersion = "0.8.12" + toolVersion = "0.8.14" } tasks.jacocoTestReport { From c694a0400e9916d4b76e77f452167e31b807c59b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 21 May 2026 14:19:31 +0200 Subject: [PATCH 06/17] chore: update backend dependencies --- build.gradle.kts | 12 ++++++------ gradle/wrapper/gradle-wrapper.properties | 2 +- .../dev/typetype/server/routes/RestoreRoutes.kt | 4 ++-- .../server/routes/YoutubeTakeoutImportRoutes.kt | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 1e35339..d102bed 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,7 @@ plugins { kotlin("jvm") version "2.3.21" kotlin("plugin.serialization") version "2.3.21" - id("io.ktor.plugin") version "3.4.3" + id("io.ktor.plugin") version "3.5.0" id("jacoco") } @@ -29,17 +29,17 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.5.32") - implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:b7109238ab38e16272ea17d925d41a6292bce014") + implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:a69bcc15d146d391a695210a52cbde7b3fff1137") implementation("com.squareup.okhttp3:okhttp:5.3.2") - implementation("io.lettuce:lettuce-core:7.5.1.RELEASE") - implementation("org.jetbrains.exposed:exposed-core:1.2.0") - implementation("org.jetbrains.exposed:exposed-jdbc:1.2.0") + implementation("io.lettuce:lettuce-core:7.5.2.RELEASE") + implementation("org.jetbrains.exposed:exposed-core:1.3.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.3.0") implementation("com.zaxxer:HikariCP:7.0.2") implementation("org.postgresql:postgresql:42.7.11") implementation("org.xerial:sqlite-jdbc:3.53.1.0") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.5.2") - testImplementation("org.junit.jupiter:junit-jupiter:6.0.3") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.9") testImplementation("io.ktor:ktor-server-test-host-jvm") diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37f78a6..5dd3c01 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.3.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.1-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt index 657b24a..86033ae 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt @@ -31,14 +31,14 @@ fun Route.restoreRoutes(restoreService: PipePipeBackupImporterService, authServi val part = multipart.readPart() ?: break if (part is PartData.FileItem && part.name == "file") { if (!PipePipeBackupValidators.isZipFilePart(part)) { - part.dispose() + part.release() return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid backup file type")) } PipePipeBackupUploadWriter.writeWithLimit(part.provider(), tmp, PipePipeBackupLimits.MAX_UPLOAD_BYTES) hasFile = true fileCount += 1 } - part.dispose() + part.release() } if (fileCount > 1) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Only one backup file is allowed")) diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt index 0ed03bb..a80ea8e 100644 --- a/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt @@ -32,13 +32,13 @@ fun Route.youtubeTakeoutImportRoutes( val part = multipart.readPart() ?: break if (part is PartData.FileItem && part.name == "archive") { if (!part.originalFileName.orEmpty().endsWith(".zip", true)) { - part.dispose() + part.release() return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid archive file type")) } YoutubeTakeoutUploadWriter.writeWithLimit(part.provider(), tmp, maxUploadBytes) hasFile = true } - part.dispose() + part.release() } if (!hasFile) return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing archive part")) call.respond(HttpStatusCode.Created, importService.create(userId, tmp)) From a59f7973bf03b67d7aa850b19aee4333b849c518 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 21 May 2026 14:19:49 +0200 Subject: [PATCH 07/17] docs: add OpenAPI schemas --- openapi/components/common.yaml | 24 ++++++++++++ openapi/components/media.yaml | 65 +++++++++++++++++++++++++++++++ openapi/components/streams.yaml | 68 +++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 openapi/components/common.yaml create mode 100644 openapi/components/media.yaml create mode 100644 openapi/components/streams.yaml diff --git a/openapi/components/common.yaml b/openapi/components/common.yaml new file mode 100644 index 0000000..d40be44 --- /dev/null +++ b/openapi/components/common.yaml @@ -0,0 +1,24 @@ +HealthResponse: + type: object + required: [status] + properties: + status: { type: string, example: ok } +ErrorResponse: + type: object + required: [error] + properties: + error: { type: string } + code: { type: string, default: error } + requestId: { type: string, nullable: true } +RequestIdHeader: + description: Request correlation id returned by TypeType-Server. + schema: { type: string } +JsonError: + description: Request failed. + headers: + X-Request-ID: + $ref: '#/RequestIdHeader' + content: + application/json: + schema: + $ref: '#/ErrorResponse' diff --git a/openapi/components/media.yaml b/openapi/components/media.yaml new file mode 100644 index 0000000..6360e38 --- /dev/null +++ b/openapi/components/media.yaml @@ -0,0 +1,65 @@ +VideoItem: + type: object + required: [id, title, url, thumbnailUrl, uploaderName, uploaderUrl, duration, viewCount, uploadDate, streamType, isShortFormContent, uploaderVerified] + properties: + id: { type: string } + title: { type: string } + url: { type: string } + thumbnailUrl: { type: string } + uploaderName: { type: string } + uploaderUrl: { type: string } + uploaderAvatarUrl: { type: string } + duration: { type: integer, format: int64 } + viewCount: { type: integer, format: int64 } + uploadDate: { type: string } + uploaded: { type: integer, format: int64, default: -1 } + streamType: { type: string } + isShortFormContent: { type: boolean } + uploaderVerified: { type: boolean } + shortDescription: { type: string, nullable: true } + publishedAt: { type: integer, format: int64, nullable: true } +SearchPageResponse: + type: object + required: [items, nextpage, searchSuggestion, isCorrectedSearch] + properties: + items: { type: array, items: { $ref: '#/VideoItem' } } + nextpage: { type: string, nullable: true } + searchSuggestion: { type: string, nullable: true } + isCorrectedSearch: { type: boolean } +ChannelResponse: + type: object + required: [name, description, avatarUrl, bannerUrl, subscriberCount, isVerified, videos, nextpage] + properties: + name: { type: string } + description: { type: string } + avatarUrl: { type: string } + bannerUrl: { type: string } + subscriberCount: { type: integer, format: int64 } + isVerified: { type: boolean } + videos: { type: array, items: { $ref: '#/VideoItem' } } + nextpage: { type: string, nullable: true } +CommentItem: + type: object + required: [id, text, author, authorUrl, authorAvatarUrl, likeCount, textualLikeCount, publishedTime, isHeartedByUploader, isPinned, uploaderVerified, replyCount, repliesPage] + properties: + id: { type: string } + text: { type: string } + author: { type: string } + authorUrl: { type: string } + authorAvatarUrl: { type: string } + likeCount: { type: integer, format: int64 } + textualLikeCount: { type: string } + publishedTime: { type: string } + publishedAt: { type: integer, format: int64, nullable: true } + isHeartedByUploader: { type: boolean } + isPinned: { type: boolean } + uploaderVerified: { type: boolean } + replyCount: { type: integer } + repliesPage: { type: string, nullable: true } +CommentsPageResponse: + type: object + required: [comments, nextpage, commentsDisabled] + properties: + comments: { type: array, items: { $ref: '#/CommentItem' } } + nextpage: { type: string, nullable: true } + commentsDisabled: { type: boolean } diff --git a/openapi/components/streams.yaml b/openapi/components/streams.yaml new file mode 100644 index 0000000..5f0ae01 --- /dev/null +++ b/openapi/components/streams.yaml @@ -0,0 +1,68 @@ +VideoStreamItem: + type: object + required: [url, mimeType, format, resolution, isVideoOnly, itag, width, height, fps, contentLength, initStart, initEnd, indexStart, indexEnd] + properties: + url: { type: string } + mimeType: { type: string } + format: { type: string } + resolution: { type: string } + bitrate: { type: integer, nullable: true } + codec: { type: string, nullable: true } + isVideoOnly: { type: boolean } + itag: { type: integer } + width: { type: integer } + height: { type: integer } + fps: { type: integer } + contentLength: { type: integer, format: int64 } + initStart: { type: integer, format: int64 } + initEnd: { type: integer, format: int64 } + indexStart: { type: integer, format: int64 } + indexEnd: { type: integer, format: int64 } +AudioStreamItem: + type: object + required: [url, mimeType, format, itag, contentLength, initStart, initEnd, indexStart, indexEnd, isOriginal] + properties: + url: { type: string } + mimeType: { type: string } + format: { type: string } + bitrate: { type: integer, nullable: true } + codec: { type: string, nullable: true } + quality: { type: string, nullable: true } + itag: { type: integer } + contentLength: { type: integer, format: int64 } + initStart: { type: integer, format: int64 } + initEnd: { type: integer, format: int64 } + indexStart: { type: integer, format: int64 } + indexEnd: { type: integer, format: int64 } + audioTrackId: { type: string, nullable: true } + audioTrackName: { type: string, nullable: true } + audioLocale: { type: string, nullable: true } + isOriginal: { type: boolean } +StreamResponse: + type: object + required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] + properties: + id: { type: string } + title: { type: string } + uploaderName: { type: string } + uploaderUrl: { type: string } + uploaderAvatarUrl: { type: string } + thumbnailUrl: { type: string } + description: { type: string } + duration: { type: integer, format: int64 } + viewCount: { type: integer, format: int64 } + likeCount: { type: integer, format: int64 } + dislikeCount: { type: integer, format: int64 } + uploadDate: { type: string } + uploaded: { type: integer, format: int64 } + publishedAt: { type: integer, format: int64, nullable: true } + streamType: { type: string } + isShortFormContent: { type: boolean } + originalAudioTrackId: { type: string, nullable: true } + preferredDefaultAudioTrackId: { type: string, nullable: true } + videoStreams: { type: array, items: { $ref: '#/VideoStreamItem' } } + audioStreams: { type: array, items: { $ref: '#/AudioStreamItem' } } + videoOnlyStreams: { type: array, items: { $ref: '#/VideoStreamItem' } } + subtitles: { type: array, items: { type: object }, default: [] } + relatedStreams: { type: array, items: { $ref: './media.yaml#/VideoItem' } } + sponsorBlockSegments: { type: array, items: { type: object }, default: [] } From c8e8ebc7bfaa3d618ec557f9b92264df6f2518ba Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 21 May 2026 14:20:01 +0200 Subject: [PATCH 08/17] docs: add extraction OpenAPI paths --- openapi/paths/channel.yaml | 27 +++++++++++++++++++++ openapi/paths/comments.yaml | 48 +++++++++++++++++++++++++++++++++++++ openapi/paths/health.yaml | 14 +++++++++++ openapi/paths/search.yaml | 31 ++++++++++++++++++++++++ openapi/paths/streams.yaml | 23 ++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 openapi/paths/channel.yaml create mode 100644 openapi/paths/comments.yaml create mode 100644 openapi/paths/health.yaml create mode 100644 openapi/paths/search.yaml create mode 100644 openapi/paths/streams.yaml diff --git a/openapi/paths/channel.yaml b/openapi/paths/channel.yaml new file mode 100644 index 0000000..39a24de --- /dev/null +++ b/openapi/paths/channel.yaml @@ -0,0 +1,27 @@ +Channel: + get: + tags: [extraction] + summary: Get channel metadata and videos + parameters: + - name: url + in: query + required: true + schema: { type: string } + - name: nextpage + in: query + required: false + schema: { type: string } + responses: + '200': + description: Channel metadata and video page. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/media.yaml#/ChannelResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError diff --git a/openapi/paths/comments.yaml b/openapi/paths/comments.yaml new file mode 100644 index 0000000..32c96da --- /dev/null +++ b/openapi/paths/comments.yaml @@ -0,0 +1,48 @@ +Comments: + get: + tags: [extraction] + summary: Get comments for a stream + parameters: + - name: url + in: query + required: true + schema: { type: string } + - name: nextpage + in: query + required: false + schema: { type: string } + responses: + '200': + description: Comment page. + content: + application/json: + schema: + $ref: ../components/media.yaml#/CommentsPageResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError +Replies: + get: + tags: [extraction] + summary: Get replies for a comment thread + parameters: + - name: url + in: query + required: true + schema: { type: string } + - name: repliesPage + in: query + required: true + schema: { type: string } + responses: + '200': + description: Reply page. + content: + application/json: + schema: + $ref: ../components/media.yaml#/CommentsPageResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError diff --git a/openapi/paths/health.yaml b/openapi/paths/health.yaml new file mode 100644 index 0000000..b45c8c2 --- /dev/null +++ b/openapi/paths/health.yaml @@ -0,0 +1,14 @@ +Health: + get: + tags: [health] + summary: Server liveness check + responses: + '200': + description: Server is alive. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/HealthResponse diff --git a/openapi/paths/search.yaml b/openapi/paths/search.yaml new file mode 100644 index 0000000..0dea783 --- /dev/null +++ b/openapi/paths/search.yaml @@ -0,0 +1,31 @@ +Search: + get: + tags: [extraction] + summary: Search videos by service + parameters: + - name: q + in: query + required: true + schema: { type: string } + - name: service + in: query + required: true + schema: { type: integer, enum: [0, 3, 4, 5, 6] } + - name: nextpage + in: query + required: false + schema: { type: string } + responses: + '200': + description: Search page. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/media.yaml#/SearchPageResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml new file mode 100644 index 0000000..b439167 --- /dev/null +++ b/openapi/paths/streams.yaml @@ -0,0 +1,23 @@ +Streams: + get: + tags: [extraction] + summary: Extract full stream metadata + parameters: + - name: url + in: query + required: true + schema: { type: string } + responses: + '200': + description: Stream metadata and playable streams. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/streams.yaml#/StreamResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError From 6faedb12005da019687bab725e2ac2341ce1b723 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 21 May 2026 14:20:13 +0200 Subject: [PATCH 09/17] docs: add downloader OpenAPI paths --- openapi/components/downloader.yaml | 83 ++++++++++++++++++++++ openapi/paths/downloader.yaml | 108 +++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 openapi/components/downloader.yaml create mode 100644 openapi/paths/downloader.yaml diff --git a/openapi/components/downloader.yaml b/openapi/components/downloader.yaml new file mode 100644 index 0000000..ce104da --- /dev/null +++ b/openapi/components/downloader.yaml @@ -0,0 +1,83 @@ +CreateDownloadJobRequest: + type: object + required: [url] + properties: + url: { type: string } + options: + $ref: '#/DownloadOptions' +DownloadOptions: + type: object + properties: + mode: { type: string, enum: [video, audio, thumbnail] } + quality: { type: string } + format: { type: string } + videoItag: { type: string, nullable: true } + audioItag: { type: string, nullable: true } + height: { type: integer, nullable: true } + fps: { type: integer, nullable: true } + videoCodec: { type: string, nullable: true } + audioCodec: { type: string, nullable: true } + bitrate: { type: integer, nullable: true } + allowQualityFallback: { type: boolean, default: true } + sponsorBlock: { type: boolean, default: false } + sponsorBlockCategories: { type: array, items: { type: string } } + thumbnailOnly: { type: boolean, default: false } + subtitles: + $ref: '#/SubtitleOptions' +SubtitleOptions: + type: object + properties: + enabled: { type: boolean } + auto: { type: boolean } + embed: { type: boolean } + languages: { type: array, items: { type: string } } + format: { type: string } +ResolvedDownload: + type: object + properties: + videoItag: { type: string, nullable: true } + audioItag: { type: string, nullable: true } + height: { type: integer, nullable: true } + fps: { type: integer, nullable: true } + videoCodec: { type: string, nullable: true } + audioCodec: { type: string, nullable: true } + container: { type: string, nullable: true } + bitrate: { type: integer, nullable: true } + fileName: { type: string, nullable: true } +DownloadJob: + type: object + required: [id, url, status] + properties: + id: { type: string } + url: { type: string } + status: { type: string, enum: [queued, running, done, failed] } + title: { type: string, nullable: true } + resolved: + $ref: '#/ResolvedDownload' + progressPercent: { type: integer, minimum: 0, maximum: 100, nullable: true } + downloadedBytes: { type: integer, format: int64, nullable: true } + totalBytes: { type: integer, format: int64, nullable: true } + speedBytesPerSecond: { type: integer, format: int64, nullable: true } + etaSeconds: { type: integer, nullable: true } + stage: { type: string, nullable: true, enum: [queued, extract, download, mux, done, failed, null] } + queuedAt: { type: string, format: date-time, nullable: true } + startedAt: { type: string, format: date-time, nullable: true } + finishedAt: { type: string, format: date-time, nullable: true } + queueWaitMs: { type: integer, format: int64, nullable: true } + runTimeMs: { type: integer, format: int64, nullable: true } + errorCode: { type: string, nullable: true } + error: { type: string, nullable: true } +DownloaderHealth: + type: object + required: [status] + additionalProperties: true + properties: + service: { type: string } + status: { type: string, example: ok } +DownloaderError: + type: object + required: [error] + additionalProperties: true + properties: + error: { type: string } + code: { type: string, nullable: true } diff --git a/openapi/paths/downloader.yaml b/openapi/paths/downloader.yaml new file mode 100644 index 0000000..5b061ce --- /dev/null +++ b/openapi/paths/downloader.yaml @@ -0,0 +1,108 @@ +Health: + get: + tags: [downloader] + summary: Downloader liveness check + responses: + '200': + description: Downloader is alive. + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/DownloaderHealth } +DeepHealth: + get: + tags: [downloader] + summary: Downloader dependency health check + responses: + '200': + description: Downloader dependencies are healthy. + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/DownloaderHealth } +Jobs: + post: + tags: [downloader] + summary: Create a download job + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/CreateDownloadJobRequest } + responses: + '201': + description: Job created. + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/DownloadJob } + '400': { $ref: ../components/common.yaml#/JsonError } +Job: + parameters: + - name: id + in: path + required: true + schema: { type: string } + get: + tags: [downloader] + summary: Get a download job + responses: + '200': + description: Job status. + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/DownloadJob } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [downloader] + summary: Delete a completed download job + responses: + '204': { description: Job deleted. } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } +JobEvents: + parameters: + - name: id + in: path + required: true + schema: { type: string } + get: + tags: [downloader] + summary: Stream download progress events + responses: + '200': + description: SSE stream with progress events whose data payload is a DownloadJob JSON object. + content: + text/event-stream: + schema: { type: string } +JobArtifact: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - name: Range + in: header + required: false + schema: { type: string } + get: + tags: [downloader] + summary: Download or redirect to a completed artifact + responses: + '200': { description: Artifact bytes. } + '206': { description: Partial artifact bytes. } + '302': { description: Redirect to public artifact storage. } + '404': { $ref: ../components/common.yaml#/JsonError } +JobCancel: + parameters: + - name: id + in: path + required: true + schema: { type: string } + post: + tags: [downloader] + summary: Cancel a queued or running download job + responses: + '202': + description: Cancellation accepted. + content: + application/json: + schema: { $ref: ../components/downloader.yaml#/DownloadJob } + '404': { $ref: ../components/common.yaml#/JsonError } From 899975a09e23b23095d1da448e57ed5eafb812a9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 21 May 2026 14:20:36 +0200 Subject: [PATCH 10/17] chore: add OpenAPI validation --- .github/workflows/openapi.yml | 23 +++++++++++ build.gradle.kts | 2 + gradle/openapi-validation.gradle.kts | 45 ++++++++++++++++++++++ openapi.yaml | 57 ++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 .github/workflows/openapi.yml create mode 100644 gradle/openapi-validation.gradle.kts create mode 100644 openapi.yaml diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 0000000..37ede99 --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,23 @@ +name: OpenAPI + +on: + push: + branches: ["dev"] + pull_request: + branches: ["dev", "main"] + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + java-version: "21" + distribution: temurin + - uses: gradle/actions/setup-gradle@v6 + - name: Validate OpenAPI + run: ./gradlew validateOpenApi diff --git a/build.gradle.kts b/build.gradle.kts index d102bed..3ab906f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,8 @@ plugins { id("jacoco") } +apply(from = "gradle/openapi-validation.gradle.kts") + group = "dev.typetype" version = "0.0.1" diff --git a/gradle/openapi-validation.gradle.kts b/gradle/openapi-validation.gradle.kts new file mode 100644 index 0000000..4ea3ecf --- /dev/null +++ b/gradle/openapi-validation.gradle.kts @@ -0,0 +1,45 @@ +import java.net.URLClassLoader +import org.gradle.api.GradleException + +val openApiValidator by configurations.creating + +dependencies { + openApiValidator("io.swagger.parser.v3:swagger-parser-v3:2.1.42") + openApiValidator("org.slf4j:slf4j-nop:2.0.17") +} + +tasks.register("validateOpenApi") { + val specFile = layout.projectDirectory.file("openapi.yaml") + val specFiles = layout.projectDirectory.asFileTree.matching { + include("openapi.yaml") + include("openapi/**/*.yaml") + } + inputs.files(specFiles) + doLast { + URLClassLoader(openApiValidator.resolve().map { it.toURI().toURL() }.toTypedArray()).use { loader -> + val optionsClass = loader.loadClass("io.swagger.v3.parser.core.models.ParseOptions") + val options = optionsClass.getConstructor().newInstance() + optionsClass.getMethod("setResolve", Boolean::class.javaPrimitiveType).invoke(options, true) + optionsClass.getMethod("setResolveFully", Boolean::class.javaPrimitiveType).invoke(options, true) + + val parserClass = loader.loadClass("io.swagger.v3.parser.OpenAPIV3Parser") + val result = parserClass + .getConstructor() + .newInstance() + .let { parser -> + parserClass + .getMethod("readLocation", String::class.java, MutableList::class.java, optionsClass) + .invoke(parser, specFile.asFile.absolutePath, mutableListOf(), options) + } + val messages = result.javaClass.getMethod("getMessages").invoke(result) as List<*> + val openApi = result.javaClass.getMethod("getOpenAPI").invoke(result) + if (openApi == null || messages.isNotEmpty()) { + throw GradleException(messages.joinToString(separator = "\n", prefix = "Invalid OpenAPI spec:\n")) + } + } + } +} + +tasks.named("check") { + dependsOn("validateOpenApi") +} diff --git a/openapi.yaml b/openapi.yaml new file mode 100644 index 0000000..b37f97d --- /dev/null +++ b/openapi.yaml @@ -0,0 +1,57 @@ +openapi: 3.0.3 +info: + title: TypeType Server API + version: 0.1.0 + description: Manual API contract for TypeType-Server. This file is maintained by hand and is not generated. +servers: + - url: http://localhost:8080 + description: Local server + - url: https://watch.eltux.fr/api + description: Production frontend proxy +tags: + - name: health + - name: extraction + - name: downloader +paths: + /health: + $ref: ./openapi/paths/health.yaml#/Health + /streams: + $ref: ./openapi/paths/streams.yaml#/Streams + /search: + $ref: ./openapi/paths/search.yaml#/Search + /channel: + $ref: ./openapi/paths/channel.yaml#/Channel + /comments: + $ref: ./openapi/paths/comments.yaml#/Comments + /comments/replies: + $ref: ./openapi/paths/comments.yaml#/Replies + /downloader/health: + $ref: ./openapi/paths/downloader.yaml#/Health + /downloader/health/deep: + $ref: ./openapi/paths/downloader.yaml#/DeepHealth + /downloader/jobs: + $ref: ./openapi/paths/downloader.yaml#/Jobs + /downloader/jobs/{id}: + $ref: ./openapi/paths/downloader.yaml#/Job + /downloader/jobs/{id}/events: + $ref: ./openapi/paths/downloader.yaml#/JobEvents + /downloader/jobs/{id}/artifact: + $ref: ./openapi/paths/downloader.yaml#/JobArtifact + /downloader/jobs/{id}/cancel: + $ref: ./openapi/paths/downloader.yaml#/JobCancel +components: + schemas: + ErrorResponse: + $ref: ./openapi/components/common.yaml#/ErrorResponse + HealthResponse: + $ref: ./openapi/components/common.yaml#/HealthResponse + StreamResponse: + $ref: ./openapi/components/streams.yaml#/StreamResponse + SearchPageResponse: + $ref: ./openapi/components/media.yaml#/SearchPageResponse + ChannelResponse: + $ref: ./openapi/components/media.yaml#/ChannelResponse + CommentsPageResponse: + $ref: ./openapi/components/media.yaml#/CommentsPageResponse + DownloadJob: + $ref: ./openapi/components/downloader.yaml#/DownloadJob From 4bb55da05e7802c7d47f03af45d0e72edcd41bc2 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 23 May 2026 20:13:14 +0200 Subject: [PATCH 11/17] chore: update extractor and silence warnings --- build.gradle.kts | 3 ++- src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt | 4 ++-- src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt | 6 +++--- .../kotlin/dev/typetype/server/routes/RestoreRoutes.kt | 2 +- src/main/resources/psw4j.properties | 7 +++++++ .../dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt | 2 +- 6 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 src/main/resources/psw4j.properties diff --git a/build.gradle.kts b/build.gradle.kts index 3ab906f..868448c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,7 +31,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.5.32") - implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:a69bcc15d146d391a695210a52cbde7b3fff1137") + implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:3e06f36ec130ca452dcf107f223eedbc30a07dc2") implementation("com.squareup.okhttp3:okhttp:5.3.2") implementation("io.lettuce:lettuce-core:7.5.2.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.3.0") @@ -74,6 +74,7 @@ tasks.test { useJUnitPlatform { excludeTags("network") } + jvmArgs("-XX:+EnableDynamicAgentLoading", "-Xshare:off") finalizedBy(tasks.jacocoTestReport) } diff --git a/src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt b/src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt index 5a46140..2d5475d 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractorLifecycle.kt @@ -16,14 +16,14 @@ private val log = LoggerFactory.getLogger("ExtractorLifecycle") internal fun CoroutineScope.launchExtractorLifecycle() { launch(Dispatchers.IO) { runCatching { StreamInfo.getInfo(WARMUP_URL) } - .onFailure { log.warn("Warmup extraction failed: ${it.message}") } + .onFailure { log.info("Warmup extraction failed: ${it.message}") } } launch(Dispatchers.IO) { while (true) { delay(THROTTLE_CLEANUP_INTERVAL_MS) runCatching { YoutubeJavaScriptPlayerManager.clearThrottlingParametersCache() } - .onFailure { log.warn("Throttling cache cleanup failed: ${it.message}") } + .onFailure { log.info("Throttling cache cleanup failed: ${it.message}") } } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt index f35e40c..0bbf93d 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt @@ -59,7 +59,7 @@ fun Route.adminRoutes( call.withAdminAuth(authService) { adminId -> val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id")) if (id == adminId) { - adminRouteLog.warn("Admin self-suspend blocked for userId={}", adminId) + adminRouteLog.info("Admin self-suspend blocked for userId={}", adminId) return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot suspend your own account")) } val ok = userAdminService.suspendUser(id) @@ -71,7 +71,7 @@ fun Route.adminRoutes( call.withAdminAuth(authService) { adminId -> val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id")) if (id == adminId) { - adminRouteLog.warn("Admin self-unsuspend blocked for userId={}", adminId) + adminRouteLog.info("Admin self-unsuspend blocked for userId={}", adminId) return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot suspend your own account")) } val ok = userAdminService.unsuspendUser(id) @@ -83,7 +83,7 @@ fun Route.adminRoutes( call.withAdminAuth(authService) { adminId -> val id = call.parameters["id"] ?: return@withAdminAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing id")) if (id == adminId) { - adminRouteLog.warn("Admin role self-change blocked for userId={}", adminId) + adminRouteLog.info("Admin role self-change blocked for userId={}", adminId) return@withAdminAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Cannot modify your own role")) } val body = runCatching { call.receive() }.getOrElse { diff --git a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt index 86033ae..645550c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt @@ -49,7 +49,7 @@ fun Route.restoreRoutes(restoreService: PipePipeBackupImporterService, authServi val result = restoreService.restore(userId, tmp, timeMode) call.respond(result) } catch (e: Exception) { - call.application.environment.log.warn("Restore backup failed", e) + call.application.environment.log.info("Restore backup failed", e) return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid backup archive")) } finally { Files.deleteIfExists(tmp) diff --git a/src/main/resources/psw4j.properties b/src/main/resources/psw4j.properties new file mode 100644 index 0000000..6317c96 --- /dev/null +++ b/src/main/resources/psw4j.properties @@ -0,0 +1,7 @@ +hash.argon2.memory=15360 +hash.argon2.iterations=2 +hash.argon2.length=32 +hash.argon2.parallelism=1 +hash.argon2.type=id +hash.argon2.version=19 +global.salt.length=64 diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt index 969d4c0..be39de1 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt @@ -7,7 +7,7 @@ import org.junit.jupiter.api.Test class YoutubeTakeoutIssueServiceTest { @Test - fun `build aggregates duplicate warnings and errors`() { + fun `build aggregates duplicate issues and errors`() { val warnings = listOf("Unsupported CSV schema: Takeout/a.csv", "No subscription rows detected", "No subscription rows detected") val errors = listOf("Invalid playlist row", "Invalid playlist row", "Invalid subscription row") val (issues, summary) = YoutubeTakeoutIssueService.build(warnings, errors, stage = "preview") From a904c9b637c3b3b699be2b45055659904d076e21 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 23 May 2026 20:13:36 +0200 Subject: [PATCH 12/17] feat: add channel tab sorting --- openapi/paths/channel.yaml | 6 ++ .../typetype/server/routes/ChannelRoutes.kt | 3 +- .../server/services/CachedChannelService.kt | 8 +- .../server/services/ChannelResponseMappers.kt | 52 +++++++++++ .../server/services/ChannelService.kt | 2 +- .../server/services/ChannelTabExtraction.kt | 16 ++++ .../server/services/PipePipeChannelService.kt | 88 ++++++------------- .../server/services/YouTubeChannelTabSort.kt | 12 +++ .../dev/typetype/server/ChannelRoutesTest.kt | 6 +- .../dev/typetype/server/FakeChannelService.kt | 2 +- 10 files changed, 122 insertions(+), 73 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt diff --git a/openapi/paths/channel.yaml b/openapi/paths/channel.yaml index 39a24de..51f542b 100644 --- a/openapi/paths/channel.yaml +++ b/openapi/paths/channel.yaml @@ -11,6 +11,12 @@ Channel: in: query required: false schema: { type: string } + - name: sort + in: query + required: false + schema: + type: string + enum: [latest, popular, oldest] responses: '200': description: Channel metadata and video page. diff --git a/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt index 568b2f8..cad9340 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt @@ -13,8 +13,9 @@ fun Route.channelRoutes(channelService: ChannelService) { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] + val sort = call.request.queryParameters["sort"]?.takeIf { it.isNotBlank() } - when (val result = channelService.getChannel(url = url, nextpage = nextpage)) { + when (val result = channelService.getChannel(url = url, nextpage = nextpage, sort = sort)) { is ExtractionResult.Success -> call.respond(result.data) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) diff --git a/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt b/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt index 8b2ab23..c31f87a 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt @@ -14,14 +14,14 @@ class CachedChannelService( private const val CHANNEL_CACHE_TTL_SECONDS = 1800L } - override suspend fun getChannel(url: String, nextpage: String?): ExtractionResult { - val key = "channel:$url:${nextpage ?: "null"}" + override suspend fun getChannel(url: String, nextpage: String?, sort: String?): ExtractionResult { + val key = "channel:$url:${nextpage ?: "null"}:${sort ?: "default"}" runCatching { cache.get(key) }.getOrNull()?.let { cached -> return runCatching { ExtractionResult.Success(CacheJson.decodeFromString(cached)) }.getOrElse { - delegate.getChannel(url, nextpage) + delegate.getChannel(url, nextpage, sort) } } - val result = delegate.getChannel(url, nextpage) + val result = delegate.getChannel(url, nextpage, sort) if (result is ExtractionResult.Success) { runCatching { cache.set(key, CacheJson.encodeToString(ChannelResponse.serializer(), result.data), CHANNEL_CACHE_TTL_SECONDS) } } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt b/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt new file mode 100644 index 0000000..0883613 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ChannelResponse +import org.schabi.newpipe.extractor.InfoItem +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage +import org.schabi.newpipe.extractor.channel.ChannelInfo +import org.schabi.newpipe.extractor.channel.ChannelTabInfo +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +internal fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( + name = name ?: "", + description = description ?: "", + avatarUrl = avatarUrl ?: "", + bannerUrl = bannerUrl ?: "", + subscriberCount = subscriberCount, + isVerified = isVerified, + videos = relatedItems.map { it.toVideoItem(fallbackAvatarUrl = avatarUrl ?: "") }, + nextpage = nextPage?.toCursor(), +) + +internal fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( + name = "", + description = "", + avatarUrl = "", + bannerUrl = "", + subscriberCount = -1L, + isVerified = false, + videos = items.map { it.toVideoItem() }, + nextpage = nextPage?.toCursor(), +) + +internal fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): ChannelResponse = ChannelResponse( + name = metadata?.name ?: name ?: "", + description = metadata?.description ?: "", + avatarUrl = metadata?.avatarUrl ?: "", + bannerUrl = metadata?.bannerUrl ?: "", + subscriberCount = metadata?.subscriberCount ?: -1L, + isVerified = metadata?.isVerified ?: false, + videos = relatedItems.filterIsInstance().map { it.toVideoItem(fallbackAvatarUrl = metadata?.avatarUrl ?: "") }, + nextpage = nextPage?.toCursor(), +) + +internal fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( + name = "", + description = "", + avatarUrl = "", + bannerUrl = "", + subscriberCount = -1L, + isVerified = false, + videos = items.filterIsInstance().map { it.toVideoItem() }, + nextpage = nextPage?.toCursor(), +) diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelService.kt b/src/main/kotlin/dev/typetype/server/services/ChannelService.kt index 5da36bc..c5dba4d 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ChannelService.kt @@ -4,5 +4,5 @@ import dev.typetype.server.models.ChannelResponse import dev.typetype.server.models.ExtractionResult interface ChannelService { - suspend fun getChannel(url: String, nextpage: String?): ExtractionResult + suspend fun getChannel(url: String, nextpage: String?, sort: String? = null): ExtractionResult } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt new file mode 100644 index 0000000..f762a12 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.channel.ChannelTabExtractor +import org.schabi.newpipe.extractor.search.filter.Filter +import org.schabi.newpipe.extractor.search.filter.FilterItem + +internal fun StreamingService.channelTabExtractor( + channelId: String, + tab: String, + sort: String?, +): ChannelTabExtractor { + val contentFilter = listOf(FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, tab)) + val linkHandler = channelTabLHFactory.fromQuery(channelId, contentFilter, sort.toYouTubeChannelTabSortFilter()) + return getChannelTabExtractor(linkHandler) +} diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt index 7689351..f53f4b1 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt @@ -5,30 +5,32 @@ import dev.typetype.server.models.ExtractionResult import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout -import org.schabi.newpipe.extractor.InfoItem -import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.Page +import org.schabi.newpipe.extractor.StreamingService import org.schabi.newpipe.extractor.channel.ChannelInfo import org.schabi.newpipe.extractor.channel.ChannelTabInfo import org.schabi.newpipe.extractor.linkhandler.ChannelTabs -import org.schabi.newpipe.extractor.stream.StreamInfoItem class PipePipeChannelService : ChannelService { - override suspend fun getChannel(url: String, nextpage: String?): ExtractionResult = + override suspend fun getChannel(url: String, nextpage: String?, sort: String?): ExtractionResult = withContext(Dispatchers.IO) { + val normalizedSort = sort?.takeIf { it.isNotBlank() } val page = if (nextpage != null) { runCatching { nextpage.toPage() } .getOrElse { return@withContext ExtractionResult.BadRequest("Invalid nextpage cursor") } } else null + runCatching { normalizedSort.toYouTubeChannelTabSortFilter() } + .getOrElse { return@withContext ExtractionResult.BadRequest(it.message ?: "Invalid 'sort' parameter") } runCatching { withExtractionRetry { withTimeout(30_000L) { if (page == null) { - extractFirstPage(url) + extractFirstPage(url, normalizedSort) } else { - extractMorePage(url, page) + extractMorePage(url, page, normalizedSort) } } } @@ -38,75 +40,35 @@ class PipePipeChannelService : ChannelService { ) } - private fun extractFirstPage(url: String): ChannelResponse { - if (isShortsTab(url)) { - val service = NewPipe.getServiceByUrl(url) - val channelId = shortsChannelId(url, service) - val extractor = service.getChannelTabExtractorFromId(channelId, ChannelTabs.SHORTS) + private fun extractFirstPage(url: String, sort: String?): ChannelResponse { + val service = NewPipe.getServiceByUrl(url) + val tab = url.toChannelTab(sort) + if (tab != null) { + val channelUrl = url.toBaseChannelUrl(tab) + val metadata = runCatching { ChannelInfo.getInfo(channelUrl) }.getOrNull() + val extractor = service.channelTabExtractor(channelId(channelUrl, service), tab, sort) extractor.fetchPage() - return ChannelTabInfo.getInfo(extractor).toChannelTabResponse() + return ChannelTabInfo.getInfo(extractor).toChannelTabResponse(metadata) } return ChannelInfo.getInfo(url).toChannelResponse() } - private fun extractMorePage(url: String, page: org.schabi.newpipe.extractor.Page): ChannelResponse { + private fun extractMorePage(url: String, page: Page, sort: String?): ChannelResponse { val service = NewPipe.getServiceByUrl(url) - if (isShortsTab(url)) { - val channelId = shortsChannelId(url, service) - val extractor = service.getChannelTabExtractorFromId(channelId, ChannelTabs.SHORTS) + val tab = url.toChannelTab(sort) + if (tab != null) { + val extractor = service.channelTabExtractor(channelId(url.toBaseChannelUrl(tab), service), tab, sort) return extractor.getPage(page).toChannelTabResponse() } return ChannelInfo.getMoreItems(service, url, page).toChannelResponse() } - private fun isShortsTab(url: String): Boolean = url.contains("/shorts", ignoreCase = true) - - private fun shortsChannelId(url: String, service: org.schabi.newpipe.extractor.StreamingService): String { - val baseUrl = url.substringBefore("/shorts").trimEnd('/') - return service.channelLHFactory.fromUrl(baseUrl).id + private fun String.toChannelTab(sort: String?): String? { + if (contains("/shorts", ignoreCase = true)) return ChannelTabs.SHORTS + return if (sort != null) ChannelTabs.VIDEOS else null } - private fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( - name = name ?: "", - description = description ?: "", - avatarUrl = avatarUrl ?: "", - bannerUrl = bannerUrl ?: "", - subscriberCount = subscriberCount, - isVerified = isVerified, - videos = relatedItems.map { it.toVideoItem(fallbackAvatarUrl = avatarUrl ?: "") }, - nextpage = nextPage?.toCursor(), - ) - - private fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( - name = "", - description = "", - avatarUrl = "", - bannerUrl = "", - subscriberCount = -1L, - isVerified = false, - videos = items.map { it.toVideoItem() }, - nextpage = nextPage?.toCursor(), - ) - - private fun ChannelTabInfo.toChannelTabResponse(): ChannelResponse = ChannelResponse( - name = name ?: "", - description = "", - avatarUrl = "", - bannerUrl = "", - subscriberCount = -1L, - isVerified = false, - videos = relatedItems.filterIsInstance().map { it.toVideoItem() }, - nextpage = nextPage?.toCursor(), - ) + private fun String.toBaseChannelUrl(tab: String): String = substringBefore("/$tab").substringBefore('?').substringBefore('#').trimEnd('/') - private fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( - name = "", - description = "", - avatarUrl = "", - bannerUrl = "", - subscriberCount = -1L, - isVerified = false, - videos = items.filterIsInstance().map { it.toVideoItem() }, - nextpage = nextPage?.toCursor(), - ) + private fun channelId(url: String, service: StreamingService): String = service.channelLHFactory.fromUrl(url).id } diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt new file mode 100644 index 0000000..525c7e5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.search.filter.Filter +import org.schabi.newpipe.extractor.search.filter.FilterItem + +private val VALID_YOUTUBE_CHANNEL_TAB_SORTS = setOf("latest", "popular", "oldest") + +internal fun String?.toYouTubeChannelTabSortFilter(): List? { + val normalized = this?.lowercase()?.takeIf { it.isNotBlank() } ?: return null + if (normalized !in VALID_YOUTUBE_CHANNEL_TAB_SORTS) throw IllegalArgumentException("Invalid 'sort' parameter") + return listOf(FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, normalized)) +} diff --git a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt index 00c1d88..d6f9cb5 100644 --- a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt @@ -48,7 +48,7 @@ class ChannelRoutesTest { @Test fun `GET channel returns 200 on Success`() = withApp { - coEvery { channelService.getChannel(any(), any()) } returns + coEvery { channelService.getChannel(any(), any(), any()) } returns ExtractionResult.Success(testChannelResponse()) val response = client.get("/channel?url=https://youtube.com/channel/test") assertEquals(HttpStatusCode.OK, response.status) @@ -56,7 +56,7 @@ class ChannelRoutesTest { @Test fun `GET channel returns 422 on Failure`() = withApp { - coEvery { channelService.getChannel(any(), any()) } returns + coEvery { channelService.getChannel(any(), any(), any()) } returns ExtractionResult.Failure("error") val response = client.get("/channel?url=https://youtube.com/channel/test") assertEquals(HttpStatusCode.UnprocessableEntity, response.status) @@ -64,7 +64,7 @@ class ChannelRoutesTest { @Test fun `GET channel returns 400 on BadRequest`() = withApp { - coEvery { channelService.getChannel(any(), any()) } returns + coEvery { channelService.getChannel(any(), any(), any()) } returns ExtractionResult.BadRequest("bad") val response = client.get("/channel?url=https://youtube.com/channel/test") assertEquals(HttpStatusCode.BadRequest, response.status) diff --git a/src/test/kotlin/dev/typetype/server/FakeChannelService.kt b/src/test/kotlin/dev/typetype/server/FakeChannelService.kt index 1bf58a9..7a330c0 100644 --- a/src/test/kotlin/dev/typetype/server/FakeChannelService.kt +++ b/src/test/kotlin/dev/typetype/server/FakeChannelService.kt @@ -6,7 +6,7 @@ import dev.typetype.server.models.VideoItem import dev.typetype.server.services.ChannelService class FakeChannelService : ChannelService { - override suspend fun getChannel(url: String, nextpage: String?): ExtractionResult { + override suspend fun getChannel(url: String, nextpage: String?, sort: String?): ExtractionResult { val video = VideoItem( id = "id-${url.hashCode()}", title = "video", From 637df04e25e31aa28ceddff03a4c4097d5b2925b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 23 May 2026 20:13:50 +0200 Subject: [PATCH 13/17] feat: add podcast extraction services --- .../server/models/PodcastEpisodesResponse.kt | 10 ++ .../dev/typetype/server/models/PodcastItem.kt | 14 +++ .../server/models/PodcastPageResponse.kt | 12 +++ .../server/services/CachedPodcastService.kt | 45 +++++++++ .../PipePipePodcastEpisodesService.kt | 52 ++++++++++ .../server/services/PipePipePodcastService.kt | 96 +++++++++++++++++++ .../server/services/PodcastItemMappers.kt | 35 +++++++ .../server/services/PodcastService.kt | 10 ++ 8 files changed, 274 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/PodcastItem.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt b/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt new file mode 100644 index 0000000..1ab1e5d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PodcastEpisodesResponse( + val podcast: PodcastItem, + val episodes: List, + val nextpage: String?, +) diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt b/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt new file mode 100644 index 0000000..4cf46b9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PodcastItem( + val id: String, + val title: String, + val url: String, + val thumbnailUrl: String, + val uploaderName: String, + val streamCount: Long, + val playlistType: String, +) diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt b/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt new file mode 100644 index 0000000..7fa1e8b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PodcastPageResponse( + val channelName: String, + val channelUrl: String, + val podcasts: List, + val episodes: List, + val nextpage: String?, +) diff --git a/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt b/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt new file mode 100644 index 0000000..53eaa19 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.PodcastEpisodesResponse +import dev.typetype.server.models.PodcastPageResponse + +class CachedPodcastService( + private val delegate: PodcastService, + private val cache: CacheService, +) : PodcastService { + + companion object { + private const val PODCAST_CACHE_TTL_SECONDS = 1800L + } + + override suspend fun getPodcasts(url: String, nextpage: String?): ExtractionResult { + val key = "podcasts:$url:${nextpage ?: "null"}" + runCatching { cache.get(key) }.getOrNull()?.let { cached -> + return runCatching { ExtractionResult.Success(CacheJson.decodeFromString(cached)) }.getOrElse { + delegate.getPodcasts(url, nextpage) + } + } + val result = delegate.getPodcasts(url, nextpage) + if (result is ExtractionResult.Success) { + runCatching { cache.set(key, CacheJson.encodeToString(PodcastPageResponse.serializer(), result.data), PODCAST_CACHE_TTL_SECONDS) } + } + return result + } + + override suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult { + val key = "podcast-episodes:$url:${nextpage ?: "null"}" + runCatching { cache.get(key) }.getOrNull()?.let { cached -> + return runCatching { ExtractionResult.Success(CacheJson.decodeFromString(cached)) }.getOrElse { + delegate.getPodcastEpisodes(url, nextpage) + } + } + val result = delegate.getPodcastEpisodes(url, nextpage) + if (result is ExtractionResult.Success) { + runCatching { cache.set(key, CacheJson.encodeToString(PodcastEpisodesResponse.serializer(), result.data), PODCAST_CACHE_TTL_SECONDS) } + } + return result + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt new file mode 100644 index 0000000..0d2579f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.PodcastEpisodesResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.playlist.PlaylistInfo +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +internal class PipePipePodcastEpisodesService { + + suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult = + withContext(Dispatchers.IO) { + val page = if (nextpage != null) { + runCatching { nextpage.toPage() } + .getOrElse { return@withContext ExtractionResult.BadRequest("Invalid nextpage cursor") } + } else null + val service = runCatching { NewPipe.getServiceByUrl(url) } + .getOrElse { return@withContext ExtractionResult.BadRequest("Unsupported podcast URL") } + if (service.serviceId != YOUTUBE_SERVICE_ID) { + return@withContext ExtractionResult.BadRequest("Podcasts are only supported for YouTube playlists") + } + + runCatching { + withExtractionRetry { + withTimeout(30_000L) { + if (page == null) PlaylistInfo.getInfo(service, url).toPodcastEpisodesResponse() + else PlaylistInfo.getMoreItems(service, url, page).toPodcastEpisodesResponse(url) + } + } + }.fold( + onSuccess = { ExtractionResult.Success(it) }, + onFailure = { ExtractionResult.Failure(it.message ?: "Podcast episodes extraction failed") } + ) + } + + private fun PlaylistInfo.toPodcastEpisodesResponse(): PodcastEpisodesResponse = PodcastEpisodesResponse( + podcast = toPodcastItem(), + episodes = relatedItems.map { it.toVideoItem() }, + nextpage = nextPage?.toCursor(), + ) + + private fun InfoItemsPage.toPodcastEpisodesResponse(url: String): PodcastEpisodesResponse = + PodcastEpisodesResponse( + podcast = emptyPodcastItem(url), + episodes = items.map { it.toVideoItem() }, + nextpage = nextPage?.toCursor(), + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt new file mode 100644 index 0000000..21be002 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.PodcastEpisodesResponse +import dev.typetype.server.models.PodcastPageResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.schabi.newpipe.extractor.InfoItem +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.Page +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.channel.ChannelTabInfo +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +class PipePipePodcastService : PodcastService { + + private val episodesService = PipePipePodcastEpisodesService() + + override suspend fun getPodcasts(url: String, nextpage: String?): ExtractionResult = + withContext(Dispatchers.IO) { + val page = if (nextpage != null) { + runCatching { nextpage.toPage() } + .getOrElse { return@withContext ExtractionResult.BadRequest("Invalid nextpage cursor") } + } else null + val channelUrl = url.toPodcastChannelUrl() + val service = runCatching { NewPipe.getServiceByUrl(channelUrl) } + .getOrElse { return@withContext ExtractionResult.BadRequest("Unsupported podcast URL") } + if (service.serviceId != YOUTUBE_SERVICE_ID) { + return@withContext ExtractionResult.BadRequest("Podcasts are only supported for YouTube channels") + } + val channelId = runCatching { service.channelLHFactory.fromUrl(channelUrl).id } + .getOrElse { return@withContext ExtractionResult.BadRequest("Unsupported podcast URL") } + + runCatching { + withExtractionRetry { + withTimeout(30_000L) { extractPodcastPage(service, channelId, channelUrl, page) } + } + }.fold( + onSuccess = { ExtractionResult.Success(it) }, + onFailure = { ExtractionResult.Failure(it.message ?: "Podcast extraction failed") } + ) + } + + override suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult = + episodesService.getPodcastEpisodes(url, nextpage) + + private fun extractPodcastPage( + service: StreamingService, + channelId: String, + channelUrl: String, + page: Page?, + ): PodcastPageResponse { + val extractor = service.channelTabExtractor(channelId, ChannelTabs.PODCASTS, null) + if (page == null) { + extractor.fetchPage() + return ChannelTabInfo.getInfo(extractor).toPodcastPageResponse(channelUrl) + } + return extractor.getPage(page).toPodcastPageResponse(channelUrl) + } + + private fun ChannelTabInfo.toPodcastPageResponse(channelUrl: String): PodcastPageResponse = + relatedItems.toPodcastPageResponse(channelUrl = channelUrl, channelName = name ?: "", nextpage = nextPage?.toCursor()) + + private fun InfoItemsPage.toPodcastPageResponse(channelUrl: String): PodcastPageResponse = + items.toPodcastPageResponse(channelUrl = channelUrl, channelName = "", nextpage = nextPage?.toCursor()) + + private fun List.toPodcastPageResponse( + channelUrl: String, + channelName: String, + nextpage: String?, + ): PodcastPageResponse { + val podcasts = filterIsInstance().map { it.toPodcastItem() } + val episodes = filterIsInstance().map { it.toVideoItem() } + val resolvedChannelName = podcasts.firstOrNull { it.uploaderName.isNotBlank() }?.uploaderName + ?: episodes.firstOrNull { it.uploaderName.isNotBlank() }?.uploaderName + ?: channelName + return PodcastPageResponse( + channelName = resolvedChannelName, + channelUrl = channelUrl, + podcasts = podcasts, + episodes = episodes, + nextpage = nextpage, + ) + } +} + +private fun String.toPodcastChannelUrl(): String { + val normalized = trim().substringBefore('?').substringBefore('#').trimEnd('/') + val marker = "/podcasts" + val index = normalized.lowercase().indexOf(marker) + return if (index == -1) normalized else normalized.substring(0, index).trimEnd('/') +} diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt b/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt new file mode 100644 index 0000000..0ca1ba2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt @@ -0,0 +1,35 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.PodcastItem +import org.schabi.newpipe.extractor.playlist.PlaylistInfo +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem + +internal fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( + id = url ?: "", + title = name ?: "", + url = url ?: "", + thumbnailUrl = thumbnailUrl.toAbsoluteUrl(), + uploaderName = uploaderName ?: "", + streamCount = streamCount, + playlistType = playlistType?.name?.lowercase() ?: "", +) + +internal fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( + id = url ?: "", + title = name ?: "", + url = url ?: "", + thumbnailUrl = thumbnailUrl.toAbsoluteUrl(), + uploaderName = uploaderName ?: "", + streamCount = streamCount, + playlistType = playlistType?.name?.lowercase() ?: "", +) + +internal fun emptyPodcastItem(url: String): PodcastItem = PodcastItem( + id = url, + title = "", + url = url, + thumbnailUrl = "", + uploaderName = "", + streamCount = -1L, + playlistType = "", +) diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastService.kt b/src/main/kotlin/dev/typetype/server/services/PodcastService.kt new file mode 100644 index 0000000..d215f26 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PodcastService.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.PodcastEpisodesResponse +import dev.typetype.server.models.PodcastPageResponse + +interface PodcastService { + suspend fun getPodcasts(url: String, nextpage: String?): ExtractionResult + suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult +} From b407cec921e116cda24c28d3fb9f95d50268efa7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 23 May 2026 20:14:04 +0200 Subject: [PATCH 14/17] feat: expose podcast routes --- .../kotlin/dev/typetype/server/Application.kt | 2 + .../dev/typetype/server/ServiceRegistry.kt | 3 + .../typetype/server/routes/PodcastRoutes.kt | 34 +++++++ .../dev/typetype/server/PodcastRoutesTest.kt | 99 +++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt create mode 100644 src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 488a34b..1b10789 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -11,6 +11,7 @@ import dev.typetype.server.routes.downloaderGatewayRoutes import dev.typetype.server.routes.internalObservabilityRoutes import dev.typetype.server.routes.manifestRoutes import dev.typetype.server.routes.nicoVideoProxyRoutes +import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.proxyRoutes import dev.typetype.server.routes.storyboardProxyRoutes import dev.typetype.server.routes.searchRoutes @@ -94,6 +95,7 @@ fun Application.module() { } rateLimit(CHANNEL_ZONE) { channelRoutes(svc.channelService) + podcastRoutes(svc.podcastService) } rateLimit(PROXY_ZONE) { proxyRoutes(svc.proxyService) diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 6100a64..0822c8e 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -8,6 +8,7 @@ import dev.typetype.server.services.CachedChannelService import dev.typetype.server.services.CachedCommentService import dev.typetype.server.services.CachedManifestService import dev.typetype.server.services.CachedNativeManifestService +import dev.typetype.server.services.CachedPodcastService import dev.typetype.server.services.CachedSearchService import dev.typetype.server.services.CachedStreamService import dev.typetype.server.services.CachedSuggestionService @@ -25,6 +26,7 @@ import dev.typetype.server.services.OkHttpProxyService import dev.typetype.server.services.PipePipeBulletCommentService import dev.typetype.server.services.PipePipeChannelService import dev.typetype.server.services.PipePipeCommentService +import dev.typetype.server.services.PipePipePodcastService import dev.typetype.server.services.PipePipeSearchService import dev.typetype.server.services.PipePipeStreamService import dev.typetype.server.services.PipePipeSuggestionService @@ -58,6 +60,7 @@ internal class ServiceRegistry(cache: DragonflyService, subtitleServiceUrl: Stri val commentService = CachedCommentService(PipePipeCommentService(), cache) val bulletCommentService = PipePipeBulletCommentService() val channelService = CachedChannelService(PipePipeChannelService(), cache) + val podcastService = CachedPodcastService(PipePipePodcastService(), cache) val proxyService = OkHttpProxyService(proxyHttpClient) val nicoVideoProxyService = NicoVideoProxyService() val manifestService = CachedManifestService(ManifestService(streamService), cache) diff --git a/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt new file mode 100644 index 0000000..2f370c8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt @@ -0,0 +1,34 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.PodcastService +import io.ktor.http.HttpStatusCode +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get + +fun Route.podcastRoutes(podcastService: PodcastService) { + get("/podcasts") { + val url = call.request.queryParameters["url"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) + val nextpage = call.request.queryParameters["nextpage"] + + when (val result = podcastService.getPodcasts(url = url, nextpage = nextpage)) { + is ExtractionResult.Success -> call.respond(result.data) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + } + } + get("/podcasts/episodes") { + val url = call.request.queryParameters["url"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) + val nextpage = call.request.queryParameters["nextpage"] + + when (val result = podcastService.getPodcastEpisodes(url = url, nextpage = nextpage)) { + is ExtractionResult.Success -> call.respond(result.data) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt new file mode 100644 index 0000000..c57a149 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt @@ -0,0 +1,99 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.PodcastEpisodesResponse +import dev.typetype.server.models.PodcastItem +import dev.typetype.server.models.PodcastPageResponse +import dev.typetype.server.routes.podcastRoutes +import dev.typetype.server.services.PodcastService +import io.ktor.client.request.get +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class PodcastRoutesTest { + + private val podcastService: PodcastService = mockk() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { podcastRoutes(podcastService) } + } + block() + } + + private fun testPodcastResponse() = PodcastPageResponse( + channelName = "Test Channel", + channelUrl = "https://youtube.com/@test", + podcasts = emptyList(), + episodes = emptyList(), + nextpage = null, + ) + + private fun testPodcastEpisodesResponse() = PodcastEpisodesResponse( + podcast = PodcastItem( + id = "https://youtube.com/playlist?list=test", + title = "Test Podcast", + url = "https://youtube.com/playlist?list=test", + thumbnailUrl = "", + uploaderName = "Test Channel", + streamCount = 1L, + playlistType = "normal", + ), + episodes = emptyList(), + nextpage = null, + ) + + @Test + fun `GET podcasts without url returns 400`() = withApp { + val response = client.get("/podcasts") + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `GET podcasts returns 200 on Success`() = withApp { + coEvery { podcastService.getPodcasts(any(), any()) } returns + ExtractionResult.Success(testPodcastResponse()) + val response = client.get("/podcasts?url=https://youtube.com/@test") + assertEquals(HttpStatusCode.OK, response.status) + } + + @Test + fun `GET podcasts returns 422 on Failure`() = withApp { + coEvery { podcastService.getPodcasts(any(), any()) } returns + ExtractionResult.Failure("error") + val response = client.get("/podcasts?url=https://youtube.com/@test") + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + } + + @Test + fun `GET podcasts returns 400 on BadRequest`() = withApp { + coEvery { podcastService.getPodcasts(any(), any()) } returns + ExtractionResult.BadRequest("bad") + val response = client.get("/podcasts?url=https://youtube.com/@test") + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `GET podcast episodes without url returns 400`() = withApp { + val response = client.get("/podcasts/episodes") + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `GET podcast episodes returns 200 on Success`() = withApp { + coEvery { podcastService.getPodcastEpisodes(any(), any()) } returns + ExtractionResult.Success(testPodcastEpisodesResponse()) + val response = client.get("/podcasts/episodes?url=https://youtube.com/playlist?list=test") + assertEquals(HttpStatusCode.OK, response.status) + } +} From df344101bf4a9e106295ef051749f9c85ba28ffb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 23 May 2026 20:14:17 +0200 Subject: [PATCH 15/17] docs: document podcast endpoints --- openapi.yaml | 8 ++++++ openapi/components/media.yaml | 27 ++++++++++++++++++ openapi/paths/podcasts.yaml | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 openapi/paths/podcasts.yaml diff --git a/openapi.yaml b/openapi.yaml index b37f97d..9e1b49a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -21,6 +21,10 @@ paths: $ref: ./openapi/paths/search.yaml#/Search /channel: $ref: ./openapi/paths/channel.yaml#/Channel + /podcasts: + $ref: ./openapi/paths/podcasts.yaml#/Podcasts + /podcasts/episodes: + $ref: ./openapi/paths/podcasts.yaml#/PodcastEpisodes /comments: $ref: ./openapi/paths/comments.yaml#/Comments /comments/replies: @@ -51,6 +55,10 @@ components: $ref: ./openapi/components/media.yaml#/SearchPageResponse ChannelResponse: $ref: ./openapi/components/media.yaml#/ChannelResponse + PodcastPageResponse: + $ref: ./openapi/components/media.yaml#/PodcastPageResponse + PodcastEpisodesResponse: + $ref: ./openapi/components/media.yaml#/PodcastEpisodesResponse CommentsPageResponse: $ref: ./openapi/components/media.yaml#/CommentsPageResponse DownloadJob: diff --git a/openapi/components/media.yaml b/openapi/components/media.yaml index 6360e38..1c5010c 100644 --- a/openapi/components/media.yaml +++ b/openapi/components/media.yaml @@ -38,6 +38,33 @@ ChannelResponse: isVerified: { type: boolean } videos: { type: array, items: { $ref: '#/VideoItem' } } nextpage: { type: string, nullable: true } +PodcastItem: + type: object + required: [id, title, url, thumbnailUrl, uploaderName, streamCount, playlistType] + properties: + id: { type: string } + title: { type: string } + url: { type: string } + thumbnailUrl: { type: string } + uploaderName: { type: string } + streamCount: { type: integer, format: int64 } + playlistType: { type: string } +PodcastPageResponse: + type: object + required: [channelName, channelUrl, podcasts, episodes, nextpage] + properties: + channelName: { type: string } + channelUrl: { type: string } + podcasts: { type: array, items: { $ref: '#/PodcastItem' } } + episodes: { type: array, items: { $ref: '#/VideoItem' } } + nextpage: { type: string, nullable: true } +PodcastEpisodesResponse: + type: object + required: [podcast, episodes, nextpage] + properties: + podcast: { $ref: '#/PodcastItem' } + episodes: { type: array, items: { $ref: '#/VideoItem' } } + nextpage: { type: string, nullable: true } CommentItem: type: object required: [id, text, author, authorUrl, authorAvatarUrl, likeCount, textualLikeCount, publishedTime, isHeartedByUploader, isPinned, uploaderVerified, replyCount, repliesPage] diff --git a/openapi/paths/podcasts.yaml b/openapi/paths/podcasts.yaml new file mode 100644 index 0000000..faa265d --- /dev/null +++ b/openapi/paths/podcasts.yaml @@ -0,0 +1,54 @@ +Podcasts: + get: + tags: [extraction] + summary: Get YouTube channel podcasts and episodes + parameters: + - name: url + in: query + required: true + schema: { type: string } + - name: nextpage + in: query + required: false + schema: { type: string } + responses: + '200': + description: YouTube podcasts tab page. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/media.yaml#/PodcastPageResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError +PodcastEpisodes: + get: + tags: [extraction] + summary: Get YouTube podcast playlist episodes + parameters: + - name: url + in: query + required: true + schema: { type: string } + - name: nextpage + in: query + required: false + schema: { type: string } + responses: + '200': + description: YouTube podcast playlist episodes. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/media.yaml#/PodcastEpisodesResponse + '400': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError From 8e9b8fb246bc330be0772a32f90a59dc7d75f142 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 4 Jun 2026 14:02:40 +0200 Subject: [PATCH 16/17] feat: add high quality playback setting --- .../dev/typetype/server/db/DatabaseFactory.kt | 1 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../server/services/SettingsService.kt | 3 +++ .../dev/typetype/server/SettingsRoutesTest.kt | 27 +++++++------------ 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index fe58b00..a4d7a7a 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -68,6 +68,7 @@ object DatabaseFactory { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_subtitle_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_audio_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS prefer_original_language BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS enable_high_quality_playback BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE history ADD COLUMN IF NOT EXISTS channel_avatar TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE history ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index f6b29fd..5a8140a 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -13,5 +13,6 @@ object SettingsTable : Table("settings") { val defaultSubtitleLanguage = text("default_subtitle_language").default("") val defaultAudioLanguage = text("default_audio_language").default("") val preferOriginalLanguage = bool("prefer_original_language").default(false) + val enableHighQualityPlayback = bool("enable_high_quality_playback").default(false) override val primaryKey = PrimaryKey(userId) } diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index baddab9..503e7c3 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -13,4 +13,5 @@ data class SettingsItem( val defaultSubtitleLanguage: String = "", val defaultAudioLanguage: String = "", val preferOriginalLanguage: Boolean = false, + val enableHighQualityPlayback: Boolean = false, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index 844b18c..bb725bd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -22,6 +22,7 @@ class SettingsService { defaultSubtitleLanguage = it[SettingsTable.defaultSubtitleLanguage], defaultAudioLanguage = it[SettingsTable.defaultAudioLanguage], preferOriginalLanguage = it[SettingsTable.preferOriginalLanguage], + enableHighQualityPlayback = it[SettingsTable.enableHighQualityPlayback], ) } ?: SettingsItem() } @@ -38,6 +39,7 @@ class SettingsService { it[defaultSubtitleLanguage] = settings.defaultSubtitleLanguage it[defaultAudioLanguage] = settings.defaultAudioLanguage it[preferOriginalLanguage] = settings.preferOriginalLanguage + it[enableHighQualityPlayback] = settings.enableHighQualityPlayback } if (updated == 0) { SettingsTable.insert { @@ -51,6 +53,7 @@ class SettingsService { it[defaultSubtitleLanguage] = settings.defaultSubtitleLanguage it[defaultAudioLanguage] = settings.defaultAudioLanguage it[preferOriginalLanguage] = settings.preferOriginalLanguage + it[enableHighQualityPlayback] = settings.enableHighQualityPlayback } } } diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index 3ef4b94..2553693 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -25,19 +25,15 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test class SettingsRoutesTest { - private val service = SettingsService() private val auth = AuthService.fixed(TEST_USER_ID) - companion object { @BeforeAll @JvmStatic fun initDb() { TestDatabase.setup() } } - @BeforeEach fun clean() { TestDatabase.truncateAll() } - private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } @@ -45,8 +41,11 @@ class SettingsRoutesTest { } block() } - private val settingsBody = """{"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false}""" + private fun assertContainsAll(body: String, values: List) = + values.forEach { assertTrue(body.contains(it)) } + private fun assertContainsNone(body: String, values: List) = + values.forEach { assertTrue(!body.contains(it)) } @Test fun `GET settings without token returns 401`() = withApp { @@ -91,12 +90,8 @@ class SettingsRoutesTest { @Test fun `GET settings returns defaults for new fields when no row exists`() = withApp { val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"subtitlesEnabled\":false")) - assertTrue(body.contains("\"defaultSubtitleLanguage\":\"\"")) - assertTrue(body.contains("\"defaultAudioLanguage\":\"\"")) - assertTrue(body.contains("\"preferOriginalLanguage\":false")) - assertTrue(!body.contains("recommendationPersonalizationEnabled")) - assertTrue(!body.contains("subscriptionSyncInterval")) + assertContainsAll(body, listOf("\"subtitlesEnabled\":false", "\"defaultSubtitleLanguage\":\"\"", "\"defaultAudioLanguage\":\"\"", "\"preferOriginalLanguage\":false", "\"enableHighQualityPlayback\":false")) + assertContainsNone(body, listOf("recommendationPersonalizationEnabled", "subscriptionSyncInterval")) } @Test @@ -104,15 +99,11 @@ class SettingsRoutesTest { client.put("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - setBody("""{"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"subtitlesEnabled":true,"defaultSubtitleLanguage":"fr","defaultAudioLanguage":"fr","preferOriginalLanguage":true,"subscriptionSyncInterval":60}""") + setBody("""{"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"subtitlesEnabled":true,"defaultSubtitleLanguage":"fr","defaultAudioLanguage":"fr","preferOriginalLanguage":true,"enableHighQualityPlayback":true,"subscriptionSyncInterval":60}""") } val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"subtitlesEnabled\":true")) - assertTrue(body.contains("\"defaultSubtitleLanguage\":\"fr\"")) - assertTrue(body.contains("\"defaultAudioLanguage\":\"fr\"")) - assertTrue(body.contains("\"preferOriginalLanguage\":true")) - assertTrue(!body.contains("recommendationPersonalizationEnabled")) - assertTrue(!body.contains("subscriptionSyncInterval")) + assertContainsAll(body, listOf("\"subtitlesEnabled\":true", "\"defaultSubtitleLanguage\":\"fr\"", "\"defaultAudioLanguage\":\"fr\"", "\"preferOriginalLanguage\":true", "\"enableHighQualityPlayback\":true")) + assertContainsNone(body, listOf("recommendationPersonalizationEnabled", "subscriptionSyncInterval")) } @Test From 8c9bc73cfe8f00f4a63f9389b8f35c8f54eef1a6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 4 Jun 2026 14:02:55 +0200 Subject: [PATCH 17/17] chore: update backend dependencies --- build.gradle.kts | 8 ++++---- gradle/openapi-validation.gradle.kts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index a5a76dd..d4d12b6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,20 +30,20 @@ dependencies { implementation("io.ktor:ktor-server-status-pages-jvm") implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") - implementation("ch.qos.logback:logback-classic:1.5.32") + implementation("ch.qos.logback:logback-classic:1.5.34") implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:871ea2df92cb81d6bc59967531523b041a9bf462") implementation("com.squareup.okhttp3:okhttp:5.3.2") - implementation("io.lettuce:lettuce-core:7.5.2.RELEASE") + implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.3.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.3.0") implementation("com.zaxxer:HikariCP:7.0.2") implementation("org.postgresql:postgresql:42.7.11") - implementation("org.xerial:sqlite-jdbc:3.53.1.0") + implementation("org.xerial:sqlite-jdbc:3.53.2.0") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.5.2") testImplementation("org.junit.jupiter:junit-jupiter:6.1.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") - testImplementation("io.mockk:mockk:1.14.9") + testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") testImplementation("io.ktor:ktor-server-content-negotiation-jvm") testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm") diff --git a/gradle/openapi-validation.gradle.kts b/gradle/openapi-validation.gradle.kts index 4ea3ecf..32c182c 100644 --- a/gradle/openapi-validation.gradle.kts +++ b/gradle/openapi-validation.gradle.kts @@ -4,7 +4,7 @@ import org.gradle.api.GradleException val openApiValidator by configurations.creating dependencies { - openApiValidator("io.swagger.parser.v3:swagger-parser-v3:2.1.42") + openApiValidator("io.swagger.parser.v3:swagger-parser-v3:2.1.43") openApiValidator("org.slf4j:slf4j-nop:2.0.17") }