diff --git a/.gitignore b/.gitignore index ccf6331..1c5b68a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,6 @@ out/ .kotlin ### Converter ### -polygon-problems sybon-packages ready src/main/resources/application.yaml diff --git a/build.gradle.kts b/build.gradle.kts index 280c927..757ac3c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -39,6 +39,8 @@ dependencies { implementation("org.jsoup:jsoup:$jsoupVersion") + implementation("org.apache.commons:commons-compress:1.26.1") // For reading zip archives from Polygon + testImplementation("io.kotest:kotest-runner-junit5:$kotestVersion") testImplementation("io.kotest:kotest-assertions-core:$kotestVersion") testImplementation("io.kotest.extensions:kotest-extensions-spring:$kotestExtensionsSpringVersion") diff --git a/frontend/src/lib/polybacs-api.ts b/frontend/src/lib/polybacs-api.ts index 7033afb..3cd0472 100644 --- a/frontend/src/lib/polybacs-api.ts +++ b/frontend/src/lib/polybacs-api.ts @@ -67,18 +67,21 @@ export function useNameAvailability( return result } +type AdditionalProperties = { + name: string + prefix: string + suffix: string + timeLimitMillis: number + memoryLimitMegabytes: number + statementFormat: string +} + export function downloadProblem({ problemId, additionalProperties, }: { problemId: number - additionalProperties: { - prefix: string - suffix: string - timeLimitMillis: number - memoryLimitMegabytes: number - statementFormat: string - } + additionalProperties: AdditionalProperties }) { fetch(`${baseUrl}/problems/${problemId}/download`, { method: 'POST', @@ -123,13 +126,7 @@ export function transferProblem({ additionalProperties, }: { problemId: number - additionalProperties: { - prefix: string - suffix: string - timeLimitMillis: number - memoryLimitMegabytes: number - statementFormat: string - } + additionalProperties: AdditionalProperties }) { fetch(`${baseUrl}/problems/${problemId}/transfer`, { method: 'POST', diff --git a/frontend/src/routes/problem.tsx b/frontend/src/routes/problem.tsx index 114541c..0bae7af 100644 --- a/frontend/src/routes/problem.tsx +++ b/frontend/src/routes/problem.tsx @@ -4,7 +4,7 @@ import { ProblemInfo, downloadProblem, getProblemInfo, - transferProblem as transferProblem, + transferProblem, useNameAvailability, } from '@/lib/polybacs-api' import { zodResolver } from '@hookform/resolvers/zod' @@ -114,7 +114,7 @@ function NameModifiersCard({ form }: { form: FormType }) { - + @@ -218,6 +218,7 @@ function Footer({ info, form }: { info: ProblemInfo; form: FormType }) { return { problemId: info.problem.id, additionalProperties: { + name: data.name, prefix: data.prefix, suffix: data.suffix, timeLimitMillis: data.timeLimitMillis, diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/api/AdditionalProblemProperties.kt b/src/main/kotlin/io/github/jvmusin/polybacs/api/AdditionalProblemProperties.kt index e0fa673..4b2c599 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/api/AdditionalProblemProperties.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/api/AdditionalProblemProperties.kt @@ -14,17 +14,13 @@ import io.github.jvmusin.polybacs.api.StatementFormat.PDF * @param statementFormat format of the statement, actually `PDF` or `HTML`. */ data class AdditionalProblemProperties( - val prefix: String? = null, + val name: String, + val prefix: String? = null, // TODO: Drop nullability; drop whole concept? val suffix: String? = null, val timeLimitMillis: Int? = null, val memoryLimitMegabytes: Int? = null, val statementFormat: StatementFormat = PDF ) { - companion object { - /** Do not add any prefix/suffix and use problem's default time and memory limits. */ - val defaultProperties = AdditionalProblemProperties() - } - /** Build problem name prefixing it with [prefix] and suffixing with [suffix] if they are not `null`. */ - fun buildFullName(problemName: String) = "${prefix.orEmpty()}$problemName${suffix.orEmpty()}" + fun buildFullName() = "${prefix.orEmpty()}$name${suffix.orEmpty()}" } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/bacs/BacsArchiveService.kt b/src/main/kotlin/io/github/jvmusin/polybacs/bacs/BacsArchiveService.kt index d60a147..57e3fd4 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/bacs/BacsArchiveService.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/bacs/BacsArchiveService.kt @@ -127,11 +127,11 @@ class BacsArchiveService( /** Uploads [problem] to Bacs archive with extra [properties]. */ suspend fun uploadProblem( problem: IRProblem, - properties: AdditionalProblemProperties = AdditionalProblemProperties.defaultProperties, + properties: AdditionalProblemProperties = AdditionalProblemProperties(problem.name), ): String { val zip = problem.toZipArchive(properties) uploadProblem(zip) - val fullName = properties.buildFullName(problem.name) + val fullName = properties.buildFullName() val state = waitTillProblemIsImported(fullName) if (state != BacsProblemState.IMPORTED) throw BacsProblemUploadException("Задача $fullName не импортирована, статус $state") diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonConfig.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonConfig.kt index 4c170fb..68664d4 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonConfig.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonConfig.kt @@ -1,12 +1,16 @@ package io.github.jvmusin.polybacs.polygon +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Configuration + /** * Polygon API configuration properties. * * @property apiKey apiKey to access the API. * @property secret secret key to access the API. */ +@Configuration data class PolygonConfig( - val apiKey: String, - val secret: String + @Value("\${polygon.apiKey}") val apiKey: String, + @Value("\${polygon.secret}") val secret: String, ) diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonModule.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonModule.kt deleted file mode 100644 index 97ca260..0000000 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonModule.kt +++ /dev/null @@ -1,26 +0,0 @@ -package io.github.jvmusin.polybacs.polygon - -import io.github.jvmusin.polybacs.polygon.api.PolygonApi -import io.github.jvmusin.polybacs.polygon.api.PolygonApiFactory -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration - -@Configuration -class PolygonModule { - @Bean - fun polygonConfig( - @Value("\${polygon.apiKey}") apiKey: String, - @Value("\${polygon.secret}") secret: String, - ) = PolygonConfig(apiKey, secret) - - @Bean - fun polygonApi(polygonConfig: PolygonConfig) = PolygonApiFactory(polygonConfig).create() - - @Bean - fun polygonProblemDownloader(polygonApi: PolygonApi) = PolygonProblemDownloaderImpl(polygonApi) - - @Bean - fun polygonService(polygonApi: PolygonApi, polygonProblemDownloader: PolygonProblemDownloader) = - PolygonService(polygonApi, polygonProblemDownloader) -} diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonProblemDownloader.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonProblemDownloader.kt index ed1e61b..7205d9d 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonProblemDownloader.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonProblemDownloader.kt @@ -25,38 +25,18 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext +import org.springframework.stereotype.Component import java.util.concurrent.ConcurrentHashMap -import kotlin.io.path.notExists -import kotlin.io.path.readText /** * Polygon problem downloader * * Used for downloading problems from Polygon. */ -interface PolygonProblemDownloader { - /** - * Downloads the problem with the given [problemId]. - * - * Tests might be skipped by setting [includeTests]. - * - * @param problemId id of the problem to download. - * @param includeTests if true then the problem tests will also be downloaded. - * @return The problem with or without tests, depending on [includeTests] parameter. - * @throws NoSuchProblemException if the problem does not exist. - * @throws AccessDeniedException if not enough rights to download the problem. - * @throws ProblemDownloadingException if something gone wrong while downloading the problem. - */ - suspend fun downloadProblem( - problemId: Int, - includeTests: Boolean, - statementFormat: StatementFormat = StatementFormat.PDF, - ): IRProblem -} - -class PolygonProblemDownloaderImpl( +@Component +class PolygonProblemDownloader( private val polygonApi: PolygonApi, -) : PolygonProblemDownloader { +) { /** * Full package id. @@ -182,13 +162,11 @@ class PolygonProblemDownloaderImpl( */ private suspend fun downloadChecker(problemId: Int, packageId: Int): IRChecker { val name = "check.cpp" - val file = polygonApi.downloadPackage(problemId, packageId).resolve(name) - if (file.notExists()) { - throw CheckerNotFoundException( + val file = polygonApi.getFileFromZipPackage(problemId, packageId, name) + ?: throw CheckerNotFoundException( "Не найден чекер '$name'. Другие чекеры не поддерживаются" ) - } - return IRChecker(name, file.readText()) + return IRChecker(name, file.decodeToString()) } /** @@ -391,45 +369,60 @@ class PolygonProblemDownloaderImpl( cache[FullPackageId(packageId, includeTests, statementFormat)] = problem } - override suspend fun downloadProblem(problemId: Int, includeTests: Boolean, statementFormat: StatementFormat) = - withContext(Dispatchers.IO) { - // eagerly check for access - val problem = getProblem(problemId) - - val packageId = polygonApi.getLatestPackageId(problemId) - - val cached = getProblemFromCache(packageId, includeTests, statementFormat) - if (cached != null) return@withContext cached - - val info = async { getProblemInfo(problemId) } - val statement = async { downloadStatement(problemId, packageId, statementFormat) } - val checker = async { downloadChecker(problemId, packageId) } - - val testsAndTestGroups = async { - /* - * These methods can throw an exception about incorrectly formatted problem, - * so throw them as soon as possible before downloading tests data. - */ - run { - info.await() - statement.await() - checker.await() - } - getTestsAndTestGroups(problemId, includeTests) + /** + * Downloads the problem with the given [problemId]. + * + * Tests might be skipped by setting [includeTests]. + * + * @param problemId id of the problem to download. + * @param includeTests if true then the problem tests will also be downloaded. + * @return The problem with or without tests, depending on [includeTests] parameter. + * @throws NoSuchProblemException if the problem does not exist. + * @throws AccessDeniedException if not enough rights to download the problem. + * @throws ProblemDownloadingException if something gone wrong while downloading the problem. + */ + suspend fun downloadProblem( + problemId: Int, + includeTests: Boolean, + statementFormat: StatementFormat = StatementFormat.PDF + ): IRProblem = withContext(Dispatchers.IO) { + // eagerly check for access + val problem = getProblem(problemId) + + val packageId = polygonApi.getLatestPackageId(problemId) + + val cached = getProblemFromCache(packageId, includeTests, statementFormat) + if (cached != null) return@withContext cached + + val info = async { getProblemInfo(problemId) } + val statement = async { downloadStatement(problemId, packageId, statementFormat) } + val checker = async { downloadChecker(problemId, packageId) } + + val testsAndTestGroups = async { + /* + * These methods can throw an exception about incorrectly formatted problem, + * so throw them as soon as possible before downloading tests data. + */ + run { + info.await() + statement.await() + checker.await() } - - val solutions = async { getSolutions(problemId) } - val limits = async { with(info.await()) { IRLimits(timeLimit, memoryLimit) } } - - IRProblem( - name = problem.name, - owner = problem.owner, - statement = statement.await(), - limits = limits.await(), - tests = testsAndTestGroups.await().first, - groups = testsAndTestGroups.await().second, - checker = checker.await(), - solutions = solutions.await() - ).also { saveProblemToCache(packageId, includeTests, statementFormat, it) } + getTestsAndTestGroups(problemId, includeTests) } + + val solutions = async { getSolutions(problemId) } + val limits = async { with(info.await()) { IRLimits(timeLimit, memoryLimit) } } + + IRProblem( + name = problem.name, + owner = problem.owner, + statement = statement.await(), + limits = limits.await(), + tests = testsAndTestGroups.await().first, + groups = testsAndTestGroups.await().second, + checker = checker.await(), + solutions = solutions.await() + ).also { saveProblemToCache(packageId, includeTests, statementFormat, it) } + } } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonService.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonService.kt index c304c0a..38a1a6b 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonService.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/PolygonService.kt @@ -6,12 +6,14 @@ import io.github.jvmusin.polybacs.polygon.api.PolygonApi import io.github.jvmusin.polybacs.polygon.exception.downloading.ProblemDownloadingException import io.github.jvmusin.polybacs.polygon.exception.response.AccessDeniedException import io.github.jvmusin.polybacs.polygon.exception.response.NoSuchProblemException +import org.springframework.stereotype.Service /** * Polygon service. * * Used to communicate to Polygon API. */ +@Service class PolygonService( private val polygonApi: PolygonApi, private val problemDownloader: PolygonProblemDownloader diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt index bd0f333..8e62fcf 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApi.kt @@ -134,7 +134,7 @@ interface PolygonApi { suspend fun getPackage( @RequestParam("problemId") problemId: Int, @RequestParam("packageId") packageId: Int, - ): ByteArray // TODO: Check if it works + ): ByteArray @PostExchange("contest.problems") suspend fun getContestProblems( diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt index 5639f49..37eb0d2 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiExtensions.kt @@ -2,28 +2,28 @@ package io.github.jvmusin.polybacs.polygon.api import io.github.jvmusin.polybacs.api.StatementFormat import io.github.jvmusin.polybacs.polygon.exception.response.NoSuchProblemException -import io.github.jvmusin.polybacs.util.extract -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.* +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.apache.commons.compress.archivers.zip.ZipFile import java.util.concurrent.ConcurrentHashMap -import java.util.zip.ZipFile -import kotlin.io.path.notExists -import kotlin.io.path.readBytes -import kotlin.io.path.writeBytes -private val packagesCache = ConcurrentHashMap() +private val packagesCache = ConcurrentHashMap() +private val packagesCacheLocks = ConcurrentHashMap() -@Suppress("BlockingMethodInNonBlockingContext") -suspend fun PolygonApi.downloadPackage(problemId: Int, packageId: Int): Path { - if (packagesCache.containsKey(packageId)) return packagesCache[packageId]!! - val destination = Paths.get("polygon-problems").resolve("id$problemId-package$packageId-${UUID.randomUUID()}") - val archivePath = Files.createTempDirectory("${destination.fileName}-").resolve("archive.zip") - archivePath.writeBytes(getPackage(problemId, packageId)) - ZipFile(archivePath.toFile()).use { it.extract(destination) } - Files.delete(archivePath) - return destination.also { packagesCache[packageId] = it } +private suspend fun PolygonApi.downloadPackageZip(problemId: Int, packageId: Int): ByteArray { + return packagesCacheLocks.computeIfAbsent(packageId) { Mutex() }.withLock { + packagesCache.getOrPut(packageId) { + getPackage(problemId, packageId) + } + } +} + +suspend fun PolygonApi.getFileFromZipPackage(problemId: Int, packageId: Int, filePath: String): ByteArray? { + val packageZipBytes = downloadPackageZip(problemId, packageId) + ZipFile.Builder().setByteArray(packageZipBytes).get().use { + val entry = it.getEntry(filePath) ?: return null + return it.getInputStream(entry).readBytes() + } } suspend fun PolygonApi.getStatementRaw( @@ -33,13 +33,13 @@ suspend fun PolygonApi.getStatementRaw( language: String = "russian", ): ByteArray? { val formatAsString = format.lowercase - val filePath = downloadPackage(problemId, packageId) - .resolve("statements") - .resolve(".$formatAsString") - .resolve(language) - .resolve("problem.$formatAsString") - if (filePath.notExists()) return null - return filePath.readBytes() + val path = listOf( + "statements", + ".$formatAsString", + language, + "problem.$formatAsString", + ).joinToString("/") + return getFileFromZipPackage(problemId, packageId, path) } /** diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt index 4cb7ef4..18f1566 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/polygon/api/PolygonApiFactory.kt @@ -2,6 +2,8 @@ package io.github.jvmusin.polybacs.polygon.api import io.github.jvmusin.polybacs.polygon.PolygonConfig import io.github.jvmusin.polybacs.util.sha512 +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.http.codec.ClientCodecConfigurer @@ -14,6 +16,7 @@ import org.springframework.web.service.invoker.createClient import org.springframework.web.util.UriComponentsBuilder import java.net.URLDecoder +@Configuration class PolygonApiFactory( private val config: PolygonConfig, ) { @@ -89,7 +92,8 @@ class PolygonApiFactory( .createClient() } - fun create(): PolygonApi { + @Bean + fun polygonApi(): PolygonApi { return createApi() } } diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/server/SolutionsController.kt b/src/main/kotlin/io/github/jvmusin/polybacs/server/SolutionsController.kt index 12f6d59..38a188b 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/server/SolutionsController.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/server/SolutionsController.kt @@ -48,8 +48,9 @@ class SolutionsController( @RequestMapping("/createTestProblem") fun createTestProblem(@PathVariable problemId: Int, session: HttpSession) = offloadScope.launch(exceptionHandler("create test problem", session)) { - val properties = AdditionalProblemProperties(suffix = "-test") - val fullName = properties.buildFullName(polygonService.downloadProblem(problemId).name) + val problemName = polygonService.downloadProblem(problemId).name + val properties = AdditionalProblemProperties(name = problemName, suffix = "-test") + val fullName = properties.buildFullName() val toastSender = webSocketConnectionKeeper.createSender(session.id) transferProblemToBacs(toastSender, problemId, properties, false, polygonService, bacsArchiveService) diff --git a/src/main/kotlin/io/github/jvmusin/polybacs/sybon/SybonArchiveBuilder.kt b/src/main/kotlin/io/github/jvmusin/polybacs/sybon/SybonArchiveBuilder.kt index 095b51a..3c74990 100644 --- a/src/main/kotlin/io/github/jvmusin/polybacs/sybon/SybonArchiveBuilder.kt +++ b/src/main/kotlin/io/github/jvmusin/polybacs/sybon/SybonArchiveBuilder.kt @@ -1,7 +1,6 @@ package io.github.jvmusin.polybacs.sybon import io.github.jvmusin.polybacs.api.AdditionalProblemProperties -import io.github.jvmusin.polybacs.api.AdditionalProblemProperties.Companion.defaultProperties import io.github.jvmusin.polybacs.ir.IRProblem import io.github.jvmusin.polybacs.util.toZipArchive import java.nio.file.Path @@ -16,8 +15,8 @@ import kotlin.io.path.writeText * * @return file where the zip is located. */ -fun IRProblem.toZipArchive(properties: AdditionalProblemProperties = defaultProperties): Path { - val fullName = properties.buildFullName(name) +fun IRProblem.toZipArchive(properties: AdditionalProblemProperties = AdditionalProblemProperties(name)): Path { + val fullName = properties.buildFullName() val destinationPath = Paths.get( "sybon-packages", "$fullName-${UUID.randomUUID()}", diff --git a/src/test/kotlin/io/github/jvmusin/polybacs/sybon/SybonSpecialCollectionTests.kt b/src/test/kotlin/io/github/jvmusin/polybacs/sybon/SybonSpecialCollectionTests.kt index 19e37cb..c39730b 100644 --- a/src/test/kotlin/io/github/jvmusin/polybacs/sybon/SybonSpecialCollectionTests.kt +++ b/src/test/kotlin/io/github/jvmusin/polybacs/sybon/SybonSpecialCollectionTests.kt @@ -23,7 +23,8 @@ class SybonSpecialCollectionTests( ) : StringSpec({ val specialCollectionId = 10023 val polygonProblemId = 147360 - val properties = AdditionalProblemProperties( + fun properties(problemName: String) = AdditionalProblemProperties( + name = problemName, suffix = LocalDateTime.now().format( DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss") ) + "-polybacs-test-rustam" @@ -31,7 +32,7 @@ class SybonSpecialCollectionTests( "Full cycle" { val irProblem = polygonService.downloadProblem(polygonProblemId, includeTests = true) - val fullName = bacsArchiveService.uploadProblem(irProblem, properties) + val fullName = bacsArchiveService.uploadProblem(irProblem, properties(irProblem.name)) println("Problem full name: $fullName") delay(2.minutes) // wait for the problem to appear in the archive val sybonProblemId = repeat(3.minutes, 10.seconds) {