diff --git a/examples/echo-kotlin/pom.xml b/examples/echo-kotlin/pom.xml index cdaba9e35f..bfb4f45207 100644 --- a/examples/echo-kotlin/pom.xml +++ b/examples/echo-kotlin/pom.xml @@ -62,26 +62,6 @@ - - - ksp - - - - - - - - com.dyescape - kotlin-maven-symbol-processing - 1.6 - - - ${project.groupId} - ftl-runtime - ${ftl.version} - - @@ -103,6 +83,13 @@ jar-with-dependencies ftl-generator.jar + + xyz.block + ftl-runtime + ${ftl.version} + jar-with-dependencies + ftl-runtime.jar + @@ -125,6 +112,7 @@ ${project.build.directory}/classpath.txt + generated.classpath dependency @@ -173,6 +161,30 @@ + + com.github.ozsie + detekt-maven-plugin + 1.23.3 + + true + ${generated.classpath} + 17 + ${project.build.directory}/detekt.yml + + + ${project.build.directory}/dependency/ftl-runtime-${ftl.version}.jar + + + + + + compile + + ctr + + + + \ No newline at end of file diff --git a/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt b/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt index c86e8ff1e8..ae420c9f1a 100644 --- a/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt +++ b/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt @@ -180,6 +180,15 @@ class ModuleGenerator() { """.trimIndent() ) + Path.of(buildDir, "detekt.yml").writeText( + """ + SchemaExtractorRuleSet: + ExtractSchemaRule: + active: true + output: ${buildDir} + """.trimIndent() + ) + val mainFile = Path.of(buildDir, "main") mainFile.writeText( """ diff --git a/kotlin-runtime/ftl-runtime/pom.xml b/kotlin-runtime/ftl-runtime/pom.xml index 5b1ff197c7..1b11406ec7 100644 --- a/kotlin-runtime/ftl-runtime/pom.xml +++ b/kotlin-runtime/ftl-runtime/pom.xml @@ -17,6 +17,7 @@ ${basedir}/../.. false + 1.23.1 @@ -32,10 +33,16 @@ 1.6.4 + + + + + + - com.google.devtools.ksp - symbol-processing-api - 1.9.20-RC-1.0.13 + io.gitlab.arturbosch.detekt + detekt-api + ${detekt.version} @@ -51,6 +58,22 @@ gson 2.10.1 + + + + io.gitlab.arturbosch.detekt + detekt-test + ${detekt.version} + + + + org.jetbrains.kotlin + kotlin-main-kts + + + test + + diff --git a/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRule.kt b/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRule.kt new file mode 100644 index 0000000000..a53eb80eab --- /dev/null +++ b/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRule.kt @@ -0,0 +1,320 @@ +package xyz.block.ftl.schemaextractor + +import io.gitlab.arturbosch.detekt.api.* +import io.gitlab.arturbosch.detekt.api.internal.RequiresTypeResolution +import io.gitlab.arturbosch.detekt.rules.fqNameOrNull +import org.jetbrains.kotlin.backend.common.serialization.metadata.findKDocString +import org.jetbrains.kotlin.cfg.getElementParentDeclaration +import org.jetbrains.kotlin.descriptors.ClassDescriptor +import org.jetbrains.kotlin.descriptors.impl.referencedProperty +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.psi.* +import org.jetbrains.kotlin.resolve.BindingContext +import org.jetbrains.kotlin.resolve.calls.util.getResolvedCall +import org.jetbrains.kotlin.resolve.calls.util.getType +import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter +import org.jetbrains.kotlin.resolve.scopes.getDescriptorsFiltered +import org.jetbrains.kotlin.resolve.source.getPsi +import org.jetbrains.kotlin.resolve.typeBinding.createTypeBindingForReturnType +import org.jetbrains.kotlin.types.KotlinType +import org.jetbrains.kotlin.types.TypeProjection +import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty +import xyz.block.ftl.Context +import xyz.block.ftl.Ignore +import xyz.block.ftl.Ingress +import xyz.block.ftl.Method +import xyz.block.ftl.schemaextractor.SchemaExtractor.Companion.moduleName +import xyz.block.ftl.v1.schema.* +import xyz.block.ftl.v1.schema.Array +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Path +import java.time.OffsetDateTime +import kotlin.Boolean +import kotlin.Long +import kotlin.String +import kotlin.collections.Map +import kotlin.collections.set +import kotlin.io.path.createDirectories + +data class ModuleData(val comments: List = emptyList(), val decls: MutableSet = mutableSetOf()) + +@RequiresTypeResolution +class ExtractSchemaRule(config: Config) : Rule(config) { + private val output: String by config(defaultValue = ".") + private val modules: MutableMap = mutableMapOf() + + override val issue = Issue( + javaClass.simpleName, + Severity.Performance, + "Verifies and extracts FTL Schema", + Debt.FIVE_MINS, + ) + + override fun visitAnnotationEntry(annotationEntry: KtAnnotationEntry) { + if ( + bindingContext.get( + BindingContext.ANNOTATION, + annotationEntry + )?.fqName?.asString() != xyz.block.ftl.Verb::class.qualifiedName + ) { + return + } + + runCatching { + val extractor = SchemaExtractor(this.bindingContext, annotationEntry) + val moduleName = annotationEntry.containingKtFile.packageFqName.moduleName() + val moduleData = extractor.extract() + modules[moduleName]?.let { it.decls += moduleData.decls } + ?: run { modules[moduleName] = moduleData } + }.onFailure { + when (it) { + is IgnoredModuleException -> return + else -> throw it + } + } + } + + override fun postVisit(root: KtFile) { + val outputDirectory = File(output).also { Path.of(it.absolutePath).createDirectories() } + + modules.toModules().forEach { + val file = File(outputDirectory.absolutePath, OUTPUT_FILENAME) + file.createNewFile() + val os = FileOutputStream(file) + os.write(it.encode()) + os.close() + } + } + + companion object { + const val OUTPUT_FILENAME = "schema.pb" + + private fun Map.toModules(): List { + return this.map { + Module(name = it.key, decls = it.value.decls.sortedBy { it.data_ == null }, comments = it.value.comments) + } + } + } +} + +class IgnoredModuleException : Exception() +class SchemaExtractor(val bindingContext: BindingContext, annotation: KtAnnotationEntry) { + private val callMatcher: Regex + private val verb: KtNamedFunction + private val module: KtDeclaration + private val decls: MutableSet = mutableSetOf() + fun extract(): ModuleData { + val requestType = requireNotNull(verb.valueParameters.last().typeReference?.resolveType()) { + "Could not resolve verb request type" + } + val responseType = requireNotNull(verb.createTypeBindingForReturnType(bindingContext)?.type) { + "Could not resolve verb response type" + } + + val metadata = mutableListOf() + extractIngress()?.apply { metadata.add(Metadata(ingress = this)) } + extractCalls()?.apply { metadata.add(Metadata(calls = this)) } + + val verb = Verb( + name = requireNotNull(verb.name) { "Verbs must be named" }, + request = requestType.toSchemaType().dataRef, + response = responseType.toSchemaType().dataRef, + metadata = metadata, + comments = verb.comments(), + ) + + decls.addAll( + listOf( + Decl(verb = verb), + Decl(data_ = requestType.toSchemaData()), + Decl(data_ = responseType.toSchemaData()) + ) + ) + + return ModuleData(decls = decls, comments = module.comments()) + } + + private fun extractIngress(): MetadataIngress? { + return verb.annotationEntries.firstOrNull { + bindingContext.get(BindingContext.ANNOTATION, it)?.fqName?.asString() == Ingress::class.qualifiedName + }?.let { + val argumentLists = it.valueArguments.partition { arg -> + // Method arg is named "method" or is of type xyz.block.ftl.Method (in the case where args are + // positional rather than named). + arg.getArgumentName()?.asName?.asString() == "method" + || arg.getArgumentExpression()?.getType(bindingContext)?.fqNameOrNull() + ?.asString() == Method::class.qualifiedName + } + val methodArg = requireNotNull(argumentLists.first.single().getArgumentExpression()?.text) { + "Could not extract method from ${verb.name} @Ingress annotation" + } + // NB: trim leading/trailing double quotes because KtStringTemplateExpression.text includes them + val pathArg = requireNotNull(argumentLists.second.single().getArgumentExpression()?.text?.trim { it == '\"' }) { + "Could not extract path from ${verb.name} @Ingress annotation" + } + MetadataIngress( + path = pathArg, + method = methodArg, + ) + } + } + + private fun extractCalls(): MetadataCalls? { + val verbs = mutableListOf() + extractCalls(verb, verbs) + return verbs.ifNotEmpty { MetadataCalls(calls = verbs) } + } + + private fun extractCalls(func: KtNamedFunction, calls: MutableList) { + val body = requireNotNull(func.bodyExpression) { "Verbs must have a body" } + val imports = func.containingKtFile.importList?.imports?.mapNotNull { it.importedFqName } ?: emptyList() + + val refs = callMatcher.findAll(body.text).map { + val req = requireNotNull(it.groups["req"]?.value?.trim()) { + "Could not extract request type for outgoing verb call from ${verb.name}" + } + val verbCall = requireNotNull(it.groups["fn"]?.value?.trim()) { + "Could not extract module name for outgoing verb call from ${verb.name}" + } + // TODO(worstell): Figure out how to get module name when not imported from another Kt file + val moduleRefName = imports.filter { it.toString().contains(req) }.firstOrNull()?.moduleName() + + VerbRef( + name = verbCall.split("::")[1].trim(), + module = moduleRefName ?: "", + ) + } + calls.addAll(refs) + + // Step into function calls inside this expression body to look for transitive calls. + body.children.mapNotNull { + (it as? KtCallExpression) + ?.getResolvedCall(bindingContext)?.candidateDescriptor?.source?.getPsi() as? KtNamedFunction + }.forEach { + extractCalls(it, calls) + } + } + + private fun KotlinType.toSchemaData(): Data { + return Data( + name = this.toClassDescriptor().name.asString(), + fields = this.memberScope.getDescriptorsFiltered(DescriptorKindFilter.VARIABLES).map { property -> + val param = requireNotNull(property.referencedProperty?.type) { "Could not resolve data class property type" } + Field( + name = property.name.asString(), + type = param.toSchemaType(param.arguments) + ) + }.toList(), + comments = this.toClassDescriptor().findKDocString()?.trim()?.let { listOf(it) } ?: emptyList() + ) + } + + private fun KotlinType.toSchemaType(typeArguments: List = emptyList()): Type { + return when (this.fqNameOrNull()?.asString()) { + String::class.qualifiedName -> Type(string = xyz.block.ftl.v1.schema.String()) + Int::class.qualifiedName -> Type(int = xyz.block.ftl.v1.schema.Int()) + Long::class.qualifiedName -> Type(int = xyz.block.ftl.v1.schema.Int()) + Boolean::class.qualifiedName -> Type(bool = xyz.block.ftl.v1.schema.Bool()) + OffsetDateTime::class.qualifiedName -> Type(time = xyz.block.ftl.v1.schema.Time()) + Map::class.qualifiedName -> { + return Type( + map = xyz.block.ftl.v1.schema.Map( + key = typeArguments.first().let { it.type.toSchemaType(it.type.getTypeArguments()) }, + value_ = typeArguments.last().let { it.type.toSchemaType(it.type.getTypeArguments()) }, + ) + ) + } + + List::class.qualifiedName -> { + return Type( + array = Array( + element = typeArguments.first().let { it.type.toSchemaType(it.type.getTypeArguments()) } + ) + ) + } + + else -> { + require( + this.toClassDescriptor().isData + && (this.fqNameOrNull()?.asString()?.startsWith("ftl.") ?: false) + ) { "${this.fqNameOrNull()?.asString()} type is not supported in FTL schema" } + + // Make sure any nested data classes are included in the module schema. + decls.add(Decl(data_ = this.toSchemaData())) + return Type( + dataRef = DataRef( + name = this.toClassDescriptor().name.asString(), + module = this.fqNameOrNull()!!.moduleName() + ) + ) + } + } + } + + private fun KtTypeReference.resolveType(): KotlinType = + bindingContext.get(BindingContext.TYPE, this) + ?: throw IllegalStateException("Could not resolve type ${this.text}") + + init { + val moduleName = annotation.containingKtFile.packageFqName.moduleName() + requireNotNull(annotation.getElementParentDeclaration()) { "Could not extract $moduleName verb definition" }.let { + require(it is KtNamedFunction) { "Verbs must be functions" } + verb = it + } + module = requireNotNull(verb.getElementParentDeclaration()) { "Could not extract $moduleName definition" } + + // Skip ignored modules. + if (module.annotationEntries.firstOrNull { + bindingContext.get(BindingContext.ANNOTATION, it)?.fqName?.asString() == Ignore::class.qualifiedName + } != null) { + throw IgnoredModuleException() + } + + requireNotNull(verb.fqName?.asString()) { + "Verbs must be defined in a package" + }.let { fqName -> + require(fqName.split(".").let { it.size >= 2 && it.first() == "ftl" }) { + "Expected @Verb to be in package ftl., but was $fqName" + } + + // Validate parameters + require(verb.valueParameters.size == 2) { "Verbs must have exactly two arguments" } + val ctxParam = verb.valueParameters.first() + val reqParam = verb.valueParameters.last() + require(ctxParam.typeReference?.resolveType()?.fqNameOrNull()?.asString() == Context::class.qualifiedName) { + "First argument of verb must be Context" + } + require(reqParam.typeReference?.resolveType()?.toClassDescriptor()?.isData ?: false) { + "Second argument of verb must be a data class" + } + + // Validate return type + val respClass = verb.createTypeBindingForReturnType(bindingContext)?.type?.toClassDescriptor() + ?: throw IllegalStateException("Could not resolve verb return type") + require(respClass.isData) { "Return type of verb must be a data class" } + + val ctxVarName = ctxParam.text.split(":")[0].trim() + callMatcher = """${ctxVarName}.call\((?[^)]+),(?[^)]+)\(\)\)""".toRegex(RegexOption.IGNORE_CASE) + } + } + + companion object { + private fun KotlinType.getTypeArguments(): List = + this.memberScope.getDescriptorsFiltered(DescriptorKindFilter.VARIABLES) + .flatMap { it.referencedProperty!!.type.arguments } + + private fun KotlinType.toClassDescriptor(): ClassDescriptor = + this.unwrap().constructor.declarationDescriptor as? ClassDescriptor + ?: throw IllegalStateException("Could not resolve KotlinType to class") + + fun FqName.moduleName(): String { + return this.asString().split(".")[1] + } + + private fun KtDeclaration.comments(): List { + return this.docComment?.text?.trim()?.let { listOf(it) } ?: emptyList() + } + } +} + diff --git a/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/SchemaExtractorRuleSetProvider.kt b/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/SchemaExtractorRuleSetProvider.kt new file mode 100644 index 0000000000..33a6f2d14f --- /dev/null +++ b/kotlin-runtime/ftl-runtime/src/main/kotlin/xyz/block/ftl/schemaextractor/SchemaExtractorRuleSetProvider.kt @@ -0,0 +1,18 @@ +package xyz.block.ftl.schemaextractor + +import io.gitlab.arturbosch.detekt.api.Config +import io.gitlab.arturbosch.detekt.api.RuleSet +import io.gitlab.arturbosch.detekt.api.RuleSetProvider + +class SchemaExtractorRuleSetProvider : RuleSetProvider { + override val ruleSetId: String = "SchemaExtractorRuleSet" + + override fun instance(config: Config): RuleSet { + return RuleSet( + ruleSetId, + listOf( + ExtractSchemaRule(config), + ), + ) + } +} diff --git a/kotlin-runtime/ftl-runtime/src/main/resources/META-INF/services/io.gitlab.arturbosch.detekt.api.RuleSetProvider b/kotlin-runtime/ftl-runtime/src/main/resources/META-INF/services/io.gitlab.arturbosch.detekt.api.RuleSetProvider new file mode 100644 index 0000000000..d21d2ffbec --- /dev/null +++ b/kotlin-runtime/ftl-runtime/src/main/resources/META-INF/services/io.gitlab.arturbosch.detekt.api.RuleSetProvider @@ -0,0 +1 @@ +xyz.block.ftl.schemaextractor.SchemaExtractorRuleSetProvider \ No newline at end of file diff --git a/kotlin-runtime/ftl-runtime/src/main/resources/config/config.yml b/kotlin-runtime/ftl-runtime/src/main/resources/config/config.yml new file mode 100644 index 0000000000..02348412a5 --- /dev/null +++ b/kotlin-runtime/ftl-runtime/src/main/resources/config/config.yml @@ -0,0 +1,3 @@ +MyRuleSet: + MyRule: + active: true \ No newline at end of file diff --git a/kotlin-runtime/ftl-runtime/src/test/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRuleTest.kt b/kotlin-runtime/ftl-runtime/src/test/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRuleTest.kt new file mode 100644 index 0000000000..4dbc3d4c92 --- /dev/null +++ b/kotlin-runtime/ftl-runtime/src/test/kotlin/xyz/block/ftl/schemaextractor/ExtractSchemaRuleTest.kt @@ -0,0 +1,230 @@ +package xyz.block.ftl.schemaextractor + +import io.gitlab.arturbosch.detekt.api.Config +import io.gitlab.arturbosch.detekt.test.compileAndLintWithContext +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import xyz.block.ftl.schemaextractor.ExtractSchemaRule.Companion.OUTPUT_FILENAME +import xyz.block.ftl.v1.schema.* +import xyz.block.ftl.v1.schema.Array +import xyz.block.ftl.v1.schema.Map +import java.io.File +import kotlin.test.AfterTest +import kotlin.test.Ignore + + +// TODO(@worstell): +// This test can't run when org.jetbrains.kotlin:kotlin-main-kts is excluded from the +// io.gitlab.arturbosch.detekt:detekt-test dependency, which is necessary to avoid a dependency conflict with +// Logback/SLF4J. When fixed we should uncomment the @KotlinCoreEnvironmentTest annotation amd no longer ignore this +// test. +@Ignore +//@KotlinCoreEnvironmentTest +internal class ExtractSchemaRuleTest(private val env: KotlinCoreEnvironment) { + + @Test + fun `extracts schema`() { + val code = """ + package ftl.echo + + import ftl.time.TimeModuleClient + import ftl.time.TimeRequest + import ftl.time.TimeResponse + import xyz.block.ftl.Context + import xyz.block.ftl.Ingress + import xyz.block.ftl.Method + import xyz.block.ftl.Verb + + class InvalidInput(val field: String) : Exception() + + data class EchoMessage(val message: String, val metadata: Map? = null) + + /** + * Request to echo a message. + */ + data class EchoRequest(val name: String) + data class EchoResponse(val messages: List) + + /** + * Echo module. + */ + class Echo { + /** + * Echoes the given message. + */ + @Throws(InvalidInput::class) + @Verb + @Ingress(Method.GET, "/echo") + fun echo(context: Context, req: EchoRequest): EchoResponse { + callTime(context) + return EchoResponse(messages = listOf(EchoMessage(message = "Hello!"))) + } + + fun callTime(context: Context): TimeResponse { + return context.call(TimeModuleClient::time, TimeRequest()) + } + } + """ + ExtractSchemaRule(Config.empty).compileAndLintWithContext(env, code) + val file = File(OUTPUT_FILENAME) + val module = Module.ADAPTER.decode(file.inputStream()) + + val expected = Module( + name = "echo", + comments = listOf( + """/** + * Echo module. + */""" + ), + decls = listOf( + Decl( + data_ = Data( + name = "EchoRequest", + fields = listOf( + Field( + name = "name", + type = Type(string = xyz.block.ftl.v1.schema.String()) + ) + ), + comments = listOf( + """/** + * Request to echo a message. + */""" + ) + ), + ), + Decl( + data_ = Data( + name = "EchoMessage", + fields = listOf( + Field( + name = "message", + type = Type(string = xyz.block.ftl.v1.schema.String()) + ), + Field( + name = "metadata", + type = Type( + map = Map( + key = xyz.block.ftl.v1.schema.Type(string = xyz.block.ftl.v1.schema.String()), + value_ = xyz.block.ftl.v1.schema.Type(string = xyz.block.ftl.v1.schema.String()) + ) + ) + ) + ) + ), + ), + Decl( + data_ = Data( + name = "EchoResponse", + fields = listOf( + Field( + name = "messages", + type = Type( + array = Array( + element = xyz.block.ftl.v1.schema.Type( + dataRef = DataRef(name = "EchoMessage", module = "echo") + ) + ) + ) + ) + ) + ), + ), + Decl( + verb = Verb( + name = "echo", + comments = listOf( + """/** + * Echoes the given message. + */""" + ), + request = DataRef( + name = "EchoRequest", + module = "echo" + ), + response = DataRef( + name = "EchoResponse", + module = "echo" + ), + metadata = listOf( + Metadata( + ingress = MetadataIngress( + method = "Method.GET", + path = "/echo" + ) + ), + Metadata( + calls = MetadataCalls( + calls = listOf( + VerbRef( + name = "time", + module = "time" + ) + ) + ) + ) + ) + ), + ) + ) + ) + + assertEquals(expected, module) + } + + @Test + fun `fails if invalid schema type is included`() { + val code = """ + package ftl.echo + + import ftl.time.TimeModuleClient + import ftl.time.TimeRequest + import ftl.time.TimeResponse + import xyz.block.ftl.Context + import xyz.block.ftl.Ingress + import xyz.block.ftl.Method + import xyz.block.ftl.Verb + + class InvalidInput(val field: String) : Exception() + + data class EchoMessage(val message: String, val metadata: Map? = null) + + /** + * Request to echo a message. + */ + data class EchoRequest(val name: Any) + data class EchoResponse(val messages: List) + + /** + * Echo module. + */ + class Echo { + /** + * Echoes the given message. + */ + @Throws(InvalidInput::class) + @Verb + @Ingress(Method.GET, "/echo") + fun echo(context: Context, req: EchoRequest): EchoResponse { + callTime(context) + return EchoResponse(messages = listOf(EchoMessage(message = "Hello!"))) + } + + fun callTime(context: Context): TimeResponse { + return context.call(TimeModuleClient::time, TimeRequest()) + } + } + """ + assertThrows(message = "kotlin.Any type is not supported in FTL schema") { + ExtractSchemaRule(Config.empty).compileAndLintWithContext(env, code) + } + } + + @AfterTest + fun cleanup() { + val file = File(OUTPUT_FILENAME) + file.delete() + } +}