From 5d8c469da235d91225ddc07fbf769abed2840a6d Mon Sep 17 00:00:00 2001 From: Rustam Musin Date: Thu, 29 Feb 2024 16:09:38 +0000 Subject: [PATCH] Migrate PolygonApi to Spring HTTP Client --- .../polybacs/polygon/api/PolygonApi.kt | 138 +++++++------- .../polygon/api/PolygonApiExtensions.kt | 2 +- .../polybacs/polygon/api/PolygonApiFactory.kt | 179 ++++++------------ .../polybacs/sybon/api/SybonApiFactory.kt | 31 +-- 4 files changed, 154 insertions(+), 196 deletions(-) diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt index 42ba3a2..2c280d0 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt @@ -1,137 +1,143 @@ package io.github.jvmusin.polybacs.polygon.api -import okhttp3.ResponseBody -import retrofit2.http.POST -import retrofit2.http.Query - +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.service.annotation.HttpExchange +import org.springframework.web.service.annotation.PostExchange + +/** + * Polygon API. + * + * Provides access to Polygon API. + */ +@HttpExchange("https://polygon.codeforces.com/api/") @Suppress("unused") interface PolygonApi { companion object { const val DEFAULT_TESTSET = "tests" } - @POST("problems.list") + @PostExchange("problems.list") suspend fun getProblems( - @Query("showDeleted") showDeleted: Boolean = false, - @Query("id") id: Int? = null, - @Query("name") name: String? = null, - @Query("owner") owner: String? = null + @RequestParam("showDeleted") showDeleted: Boolean = false, + @RequestParam("id", required = false) id: Int? = null, + @RequestParam("name", required = false) name: String? = null, + @RequestParam("owner", required = false) owner: String? = null, ): PolygonResponse> - @POST("problem.info") + @PostExchange("problem.info") suspend fun getProblemInfo( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.statements") + @PostExchange("problem.statements") suspend fun getStatements( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse> - @POST("problem.statementResources") + @PostExchange("problem.statementResources") suspend fun getStatementResources( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse> - @POST("problem.checker") + @PostExchange("problem.checker") suspend fun getCheckerName( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.validator") + @PostExchange("problem.validator") suspend fun getValidatorName( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.interactor") + @PostExchange("problem.interactor") suspend fun getInteractorName( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.files") + @PostExchange("problem.files") suspend fun getFiles( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse>> - @POST("problem.solutions") + @PostExchange("problem.solutions") suspend fun getSolutions( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse> - @POST("problem.viewFile") + @PostExchange("problem.viewFile") suspend fun getFile( - @Query("problemId") problemId: Int, - @Query("type") type: File.Type, - @Query("name") name: String - ): ResponseBody + @RequestParam("problemId") problemId: Int, + @RequestParam("type") type: File.Type, + @RequestParam("name") name: String, + ): ByteArray // TODO: Check with documentation - @POST("problem.viewSolution") + @PostExchange("problem.viewSolution") suspend fun getSolutionContent( - @Query("problemId") problemId: Int, - @Query("name") name: String + @RequestParam("problemId") problemId: Int, + @RequestParam("name") name: String, ): String - @POST("problem.script") + @PostExchange("problem.script") suspend fun getScript( - @Query("problemId") problemId: Int, - @Query("testset") name: String - ): ResponseBody + @RequestParam("problemId") problemId: Int, + @RequestParam("testset") name: String, + ): ByteArray // TODO: Check with documentation - @POST("problem.tests") + @PostExchange("problem.tests") suspend fun getTests( - @Query("problemId") problemId: Int, - @Query("testset") testSet: String = DEFAULT_TESTSET + @RequestParam("problemId") problemId: Int, + @RequestParam("testset") testSet: String = DEFAULT_TESTSET, ): PolygonResponse> - @POST("problem.testInput") + @PostExchange("problem.testInput") suspend fun getTestInput( - @Query("problemId") problemId: Int, - @Query("testIndex") testIndex: Int, - @Query("testset") testSet: String = DEFAULT_TESTSET + @RequestParam("problemId") problemId: Int, + @RequestParam("testIndex") testIndex: Int, + @RequestParam("testset") testSet: String = DEFAULT_TESTSET, ): String - @POST("problem.testAnswer") + @PostExchange("problem.testAnswer") suspend fun getTestAnswer( - @Query("problemId") problemId: Int, - @Query("testIndex") testIndex: Int, - @Query("testset") testSet: String = DEFAULT_TESTSET + @RequestParam("problemId") problemId: Int, + @RequestParam("testIndex") testIndex: Int, + @RequestParam("testset") testSet: String = DEFAULT_TESTSET, ): String - @POST("problem.viewTestGroup") + @PostExchange("problem.viewTestGroup") suspend fun getTestGroup( - @Query("problemId") problemId: Int, - @Query("group") group: String? = null, - @Query("testset") testset: String = DEFAULT_TESTSET + @RequestParam("problemId") problemId: Int, + @RequestParam("group", required = false) group: String? = null, + @RequestParam("testset") testset: String = DEFAULT_TESTSET, // TODO: Check if default value works ): PolygonResponse> - @POST("problem.viewTags") + @PostExchange("problem.viewTags") suspend fun getTags( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse> - @POST("problem.viewGeneralDescription") + @PostExchange("problem.viewGeneralDescription") suspend fun getGeneralDescription( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.viewGeneralTutorial") + @PostExchange("problem.viewGeneralTutorial") suspend fun getGeneralTutorial( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse - @POST("problem.packages") + @PostExchange("problem.packages") suspend fun getPackages( - @Query("problemId") problemId: Int + @RequestParam("problemId") problemId: Int, ): PolygonResponse> - @POST("problem.package") + @PostExchange("problem.package") suspend fun getPackage( - @Query("problemId") problemId: Int, - @Query("packageId") packageId: Int - ): ResponseBody + @RequestParam("problemId") problemId: Int, + @RequestParam("packageId") packageId: Int, + ): ByteArray // TODO: Check if it works - @POST("contest.problems") + @PostExchange("contest.problems") suspend fun getContestProblems( - @Query("contestId") contestId: Int + @RequestParam("contestId") contestId: Int, ): PolygonResponse> } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt index 966fad2..5639f49 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt @@ -20,7 +20,7 @@ suspend fun PolygonApi.downloadPackage(problemId: Int, packageId: Int): Path { if (packagesCache.containsKey(packageId)) return packagesCache[packageId]!! val destination = Paths.get("polygon-problems").resolve("id$problemId-package$packageId-${UUID.randomUUID()}") val archivePath = Files.createTempDirectory("${destination.fileName}-").resolve("archive.zip") - archivePath.writeBytes(getPackage(problemId, packageId).bytes()) + archivePath.writeBytes(getPackage(problemId, packageId)) ZipFile(archivePath.toFile()).use { it.extract(destination) } Files.delete(archivePath) return destination.also { packagesCache[packageId] = it } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt index 30d40d9..cbd472c 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt @@ -1,142 +1,89 @@ package io.github.jvmusin.polybacs.polygon.api import io.github.jvmusin.polybacs.polygon.PolygonConfig -import io.github.jvmusin.polybacs.retrofit.RetrofitClientFactory -import io.github.jvmusin.polybacs.retrofit.bufferedBody -import io.github.jvmusin.polybacs.util.RetryPolicy import io.github.jvmusin.polybacs.util.sha512 -import kotlinx.coroutines.runBlocking -import okhttp3.Interceptor -import okhttp3.Response -import org.slf4j.LoggerFactory.getLogger -import kotlin.time.Duration.Companion.minutes - -/** - * PolygonAPI factory. - * - * Used to create [PolygonApi]. - * - * @property config Polygon API configuration. - */ -class PolygonApiFactory(private val config: PolygonConfig) { +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.web.reactive.function.client.ClientRequest +import org.springframework.web.reactive.function.client.ExchangeFilterFunction +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.support.WebClientAdapter +import org.springframework.web.service.invoker.HttpServiceProxyFactory +import org.springframework.web.service.invoker.createClient +import org.springframework.web.util.UriComponentsBuilder +import java.net.URLDecoder +class PolygonApiFactory( + private val config: PolygonConfig, +) { /** - * ApiSig adding interceptor. + * Changes response code from `400` to `200`. * - * Adds *apiSig* to every request made to Polygon API. + * Used to treat code `400` responses as code `200` responses. * - * Uses [PolygonConfig.apiKey] and [PolygonConfig.secret] to change the request URL. + * Polygon API returns code `400` when something is wrong, + * but it also returns the message about that in request body, + * so we will have `null` result and `non-null` message + * in the [PolygonResponse]. */ - private inner class ApiSigAddingInterceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - val original = chain.request() - val originalUrl = original.url - - val time = System.currentTimeMillis() / 1000 - val prefix = "abcdef" - val method = originalUrl.pathSegments[1] - - val almostDoneUrl = originalUrl.newBuilder() - .addQueryParameter("apiKey", config.apiKey) - .addQueryParameter("time", time.toString()) - .build() - - val middle = almostDoneUrl.queryParameterNames - .map { it to almostDoneUrl.queryParameter(it) } - .sortedWith(compareBy({ it.first }, { it.second })) - .joinToString("&") { "${it.first}=${it.second}" } - val toHash = "$prefix/$method?$middle#${config.secret}" - val apiSig = prefix + toHash.sha512() - - val finalUrl = almostDoneUrl.newBuilder().addQueryParameter("apiSig", apiSig).build() - return chain.proceed(original.newBuilder().url(finalUrl).build()) + private val responseCode400to200Filter = ExchangeFilterFunction { request, next -> + next.exchange(request).map { + when (it.statusCode()) { + HttpStatus.BAD_REQUEST -> it.mutate().statusCode(HttpStatus.OK).build() + else -> it + } } } /** - * Polygon retry interceptor. + * Changes response content type to *application/json*. * - * Retries the request while [needRepeat] returns *true*. - * - * Duration of time it tries to repeat the request and delay between sequential requests - * are configured via [retryPolicy]. - * - * @property retryPolicy Configures duration of time it tries to repeat the request - * and delays between sequential requests. + * Polygon API returns `text/html` content type, but it's actually `application/json`. */ - private abstract class PolygonRetryInterceptor( - private val retryPolicy: RetryPolicy = RetryPolicy(10.minutes, 1.minutes), - ) : Interceptor { - abstract fun needRepeat(response: Response): Boolean - - override fun intercept(chain: Interceptor.Chain) = runBlocking { - var done = 0 - retryPolicy.evalWhileFails({ res -> - if (needRepeat(res)) { - val body = res.bufferedBody()?.string() ?: "NO BODY" - getLogger(javaClass).warn( - "Polygon API responded with code ${res.code} and body '$body'\n" + - "Now sleep for ${retryPolicy.retryAfter} and make try #${++done + 1}" - ) - false - } else true - }) { chain.proceed(chain.request()) } + private val fixResponseContentTypeFilter = ExchangeFilterFunction { request, next -> + next.exchange(request).map { + it.mutate().headers { headers -> headers.contentType = MediaType.APPLICATION_JSON }.build() } } - /** - * Too many requests retry interceptor. - * - * Retries the request if Polygon API said TOO MANY REQUESTS. - */ - private class TooManyRequestsRetryInterceptor : PolygonRetryInterceptor() { - companion object { - const val TOO_MANY_REQUESTS_MESSAGE = "Too many requests. Please, wait few seconds and try again" - } + private val insertApiSigFilter = ExchangeFilterFunction { request, next -> + val time = System.currentTimeMillis() / 1000 + val prefix = "abcdef" + val method = request.url().path.removePrefix("/api/") - override fun needRepeat(response: Response): Boolean { - return response.code == 400 && response.bufferedBody()?.string() == TOO_MANY_REQUESTS_MESSAGE - } - } + val decodedUrl = URLDecoder.decode(request.url().toString(), Charsets.UTF_8) + val builder = UriComponentsBuilder.fromUriString(decodedUrl) + .queryParam("apiKey", config.apiKey) + .queryParam("time", time.toString()) + .cloneBuilder() - /** - * Server error retry interceptor. - * - * Retries the request if Polygon API responded with 5xx code. - */ - private class ServerErrorRetryInterceptor : PolygonRetryInterceptor() { - override fun needRepeat(response: Response) = response.code in 500..599 + val middle = builder.build().queryParams + .map { it.key to it.value.single() } + .sortedWith(compareBy({ it.first }, { it.second })) + .joinToString("&") { "${it.first}=${it.second}" } + val toHash = "$prefix/$method?$middle#${config.secret}" + val apiSig = prefix + toHash.sha512() + + val finalUrl = builder.queryParam("apiSig", apiSig).build().toUri() + val newRequest = ClientRequest.from(request).url(finalUrl).build() + next.exchange(newRequest) } - /** - * Changes response code from 400 to 200. - * - * Used to treat *code 400* responses as *code 200* responses. - * - * Polygon API returns code *400* when something is wrong, - * but it also returns the message about that in request body, - * so we will have *null* result and *non-null* message - * in the [PolygonResponse]. - */ - private class Code400To200Interceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - val result = chain.proceed(chain.request()) - return when (result.code) { - 400 -> result.newBuilder().code(200).build() - else -> result - } - } + private inline fun createApi(): T { + val webClientBuilder = WebClient.builder() + .filter(fixResponseContentTypeFilter) + .filter(responseCode400to200Filter) + .filter(insertApiSigFilter) + // TODO: Add 500 and 429 retry filters + // 429 is actually 400 with text in the body + // TOO_MANY_REQUESTS_MESSAGE = "Too many requests. Please, wait few seconds and try again" + return HttpServiceProxyFactory + .builderFor(WebClientAdapter.create(webClientBuilder.build())) + .build() + .createClient() } - /** - * Creates PolygonAPI using configuration data from [config]. - * - * @return New PolygonAPI instance. - */ - fun create(): PolygonApi = RetrofitClientFactory.create("https://polygon.codeforces.com/api/") { - addInterceptor(Code400To200Interceptor()) - addInterceptor(ServerErrorRetryInterceptor()) - addInterceptor(TooManyRequestsRetryInterceptor()) - addInterceptor(ApiSigAddingInterceptor()) + fun create(): PolygonApi { + return createApi() } } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/sybon/api/SybonApiFactory.kt b/src/main/kotlin/io/github/jvmusin/polybacs/sybon/api/SybonApiFactory.kt index f5b75ce..f2ef957 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/sybon/api/SybonApiFactory.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/sybon/api/SybonApiFactory.kt @@ -1,33 +1,38 @@ package io.github.jvmusin.polybacs.sybon.api import org.springframework.beans.factory.annotation.Value +import org.springframework.http.codec.ClientCodecConfigurer import org.springframework.stereotype.Component import org.springframework.web.reactive.function.client.ClientRequest +import org.springframework.web.reactive.function.client.ExchangeFilterFunction import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.support.WebClientAdapter import org.springframework.web.service.invoker.HttpServiceProxyFactory import org.springframework.web.service.invoker.createClient import org.springframework.web.util.UriComponentsBuilder -import java.net.URLDecoder @Component class SybonApiFactory( @Value("\${sybon.apiKey}") - private val apiKey: String, + apiKey: String, ) { + private val insertApiKeyFilter = ExchangeFilterFunction { request, next -> + val newUri = UriComponentsBuilder.fromUri(request.url()) + .queryParam("api_key", apiKey) + .build(true) // do not encode + .toUri() + val newRequest = ClientRequest.from(request).url(newUri).build() + next.exchange(newRequest) + } + + private val maxInMemorySizeCodecConfigurer = { codecs: ClientCodecConfigurer -> + codecs.defaultCodecs().maxInMemorySize(16 * 1024 * 1024) // 16 MB + } + private inline fun createApi(): T { val webClientBuilder = WebClient.builder() - .filter { request, next -> - val initialUrl = URLDecoder.decode(request.url().toString(), Charsets.UTF_8) - val newUri = UriComponentsBuilder.fromUriString(initialUrl) - .queryParam("api_key", apiKey) - .build() - .toUri() - val newRequest = ClientRequest.from(request).url(newUri).build() - next.exchange(newRequest) - }.codecs { - it.defaultCodecs().maxInMemorySize(16 * 1024 * 1024) // 16 MB - } + .filter(insertApiKeyFilter) + .codecs(maxInMemorySizeCodecConfigurer) return HttpServiceProxyFactory .builderFor(WebClientAdapter.create(webClientBuilder.build())) .build()