Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pull out changes from michael/flatgraph to minify diff #4749

Merged
merged 1 commit into from
Jul 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
case Success(v) => Some(v)
case Failure(ex) =>
System.err.println("Error loading CPG")
System.err.println(ex)
ex.printStackTrace()
None
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import io.joern.dataflowengineoss.semanticsloader.Semantics
import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph}
import io.shiftleft.semanticcpg.language.*
import io.shiftleft.semanticcpg.utils.MemberAccess.isGenericMemberAccessName
import overflowdb.Node
import overflowdb.traversal.jIteratortoTraversal

import scala.collection.mutable

Expand Down Expand Up @@ -59,7 +57,7 @@ class DdgGenerator {
}
}

private def shouldBeDisplayed(v: Node): Boolean = !(
private def shouldBeDisplayed(v: StoredNode): Boolean = !(
v.isInstanceOf[ControlStructure] ||
v.isInstanceOf[JumpTarget]
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,6 @@ object Engine {
private def ddgInE(node: CfgNode, path: Vector[PathElement], callSiteStack: List[Call] = List()): Vector[Edge] = {
node
.inE(EdgeTypes.REACHING_DEF)
.asScala
.filter { e =>
e.outNode() match {
case srcNode: CfgNode =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ object SourcesToStartingPoints {
.map(src => {
// We need to get Cpg wrapper from graph. Hence we are taking head element from source iterator.
// This will also ensure if the source list is empty then these tasks are invoked.
val cpg = Cpg(src.graph())
val cpg = Cpg(src.graph)
val (startingPoints, methodTasks) = calculateStartingPoints(sources, executorService)
val startingPointFromUsageInOtherClasses =
calculateStatingPointsWithUsageInOtherClasses(methodTasks, cpg, executorService)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class TaskCreator(context: EngineContext) {
*/

private def paramToMethodRefCallReceivers(param: MethodParameterIn): List[Expression] =
new Cpg(param.graph()).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l
new Cpg(param.graph).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l

/** Create new tasks from all results that end in an output argument, including return arguments. In this case, we
* want to traverse to corresponding method output parameters and method return nodes respectively.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ class MacroHandlingTests extends C2CpgSuite {
""".stripMargin)

"should not result in malformed CFGs when expanding a nested macro with block" in {
cpg.all.collectAll[Block].l.count(b => b.cfgOut.size > 1) shouldBe 0
cpg.all.collectAll[Block].l.count(b => b._cfgOut.size > 1) shouldBe 0
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
case x: NewMethodParameterIn =>
identifierNode(dotNetNode.orNull, x.name, x.code, x.typeFullName, x.dynamicTypeHintFullName)
case x =>
logger.warn(s"Unhandled declaration type '${x.label()}' for ${x.name}")
logger.warn(s"Unhandled declaration type '${x.label}' for ${x.name}")
identifierNode(dotNetNode.orNull, x.name, x.name, Defines.Any)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@ import io.shiftleft.codepropertygraph.generated.edges.Ref
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes}
import io.shiftleft.semanticcpg.language.*
import overflowdb.traversal.{jIteratortoTraversal, toNodeTraversal}

import java.io.File

class MethodCallTests extends GoCodeToCpgSuite(withOssDataflow = true) {

"Simple method call use case" should {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -463,8 +463,8 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator =>
}

private def constructorReturnNode(constructorDeclaration: ConstructorDeclaration): NewMethodReturn = {
val line = constructorDeclaration.getEnd.map(x => x.line).toScala
val column = constructorDeclaration.getEnd.map(x => x.column).toScala
val line = constructorDeclaration.getEnd.map(_.line).toScala
val column = constructorDeclaration.getEnd.map(_.column).toScala
newMethodReturnNode(TypeConstants.Void, None, line, column)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -262,20 +262,20 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
case List(method) =>
val List(_: Local, assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked

assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
assign.name shouldBe Operators.assignment
val alloc = assign.argument(2).asInstanceOf[Call]
alloc.name shouldBe "<operator>.alloc"
alloc.code shouldBe "new Bar(4, 2)"
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
alloc.methodFullName shouldBe "<operator>.alloc"
alloc.typeFullName shouldBe "Bar"
alloc.argument.size shouldBe 0

init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
init.typeFullName shouldBe "void"
init.signature shouldBe "void(int,int)"
init.code shouldBe "new Bar(4, 2)"
Expand Down Expand Up @@ -303,20 +303,20 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
case List(method) =>
val List(assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked

assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
assign.name shouldBe Operators.assignment
val alloc = assign.argument(2).asInstanceOf[Call]
alloc.name shouldBe "<operator>.alloc"
alloc.code shouldBe "new Bar(4, 2)"
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
alloc.methodFullName shouldBe "<operator>.alloc"
alloc.typeFullName shouldBe "Bar"
alloc.argument.size shouldBe 0

init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
init.typeFullName shouldBe "void"
init.signature shouldBe "void(int,int)"
init.code shouldBe "new Bar(4, 2)"
Expand Down Expand Up @@ -362,16 +362,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
alloc.order shouldBe 2
alloc.argumentIndex shouldBe 2
alloc.code shouldBe "new Bar(42)"
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
alloc.typeFullName shouldBe "Bar"
alloc.argument.size shouldBe 0

init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.signature shouldBe "void(int)"
init.code shouldBe "new Bar(42)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()

init.argument.size shouldBe 2
val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked
Expand Down Expand Up @@ -411,16 +411,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
alloc.order shouldBe 2
alloc.argumentIndex shouldBe 2
alloc.code shouldBe "new Bar(42)"
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
alloc.typeFullName shouldBe "Bar"
alloc.argument.size shouldBe 0

init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.signature shouldBe "void(int)"
init.code shouldBe "new Bar(42)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()

init.argument.size shouldBe 2
val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked
Expand All @@ -447,7 +447,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked
init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
init.typeFullName shouldBe "void"
init.signature shouldBe "void(int)"

Expand Down Expand Up @@ -475,7 +475,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture {
val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked
init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName
init.methodFullName shouldBe s"Foo.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)"
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString
init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name()
init.typeFullName shouldBe "void"
init.signature shouldBe "void(int)"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.{
Return
}
import io.shiftleft.semanticcpg.language.*
import overflowdb.traversal.toNodeTraversal

import scala.jdk.CollectionConverters.*

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import soot.tagkit.*
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import scala.jdk.CollectionConverters.CollectionHasAsScala

trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode)
extends AstForTypeDeclsCreator
with AstForMethodsCreator { this: AstCreator =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ class MetaDataTests extends JimpleCode2CpgFixture {

"should not have any incoming or outgoing edges" in {
cpg.metaData.size shouldBe 1
cpg.metaData.in().l shouldBe List()
cpg.metaData.out().l shouldBe List()
cpg.metaData.in.l shouldBe List()
cpg.metaData.out.l shouldBe List()
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators}
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators, PropertyNames}
import ujson.Value

import scala.util.Try
Expand All @@ -29,7 +29,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this:
} else nameTpe

val astParentType = methodAstParentStack.head.label
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString

val aliasTypeDeclNode =
typeDeclNode(alias, aliasName, aliasFullName, parserResult.filename, alias.code, astParentType, astParentFullName)
Expand Down Expand Up @@ -230,7 +230,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this:
registerType(typeFullName)

val astParentType = methodAstParentStack.head.label
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unfortunately quite bad: It assembles all properties into a dictionary before extracting the one it needs.

This is not a new problem, but it does suck :(

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could define NewNode.property(name) in the codegen. after the flatgraph migration. sounds good?


val typeDeclNode_ = typeDeclNode(
tsEnum,
Expand Down Expand Up @@ -312,7 +312,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this:
registerType(typeFullName)

val astParentType = methodAstParentStack.head.label
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString

val superClass = Try(createBabelNodeInfo(clazz.json("superClass")).code).toOption.toSeq
val implements = Try(clazz.json("implements").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten
Expand Down Expand Up @@ -467,7 +467,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this:
registerType(typeFullName)

val astParentType = methodAstParentStack.head.label
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString

val extendz = Try(tsInterface.json("extends").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.DispatchTypes
import io.shiftleft.codepropertygraph.generated.Operators
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames}

trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCreator =>
protected def createMethodReturnNode(func: BabelNodeInfo): NewMethodReturn = {
Expand Down Expand Up @@ -251,7 +250,7 @@ trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstC
registerType(methodFullName)

val astParentType = parentNode.label
val astParentFullName = parentNode.properties("FULL_NAME").toString
val astParentFullName = parentNode.properties(PropertyNames.FULL_NAME).toString
val functionTypeDeclNode =
typeDeclNode(
node,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") {
val List(credentialsParam) = cpg.parameter.nameExact("credentials").l
credentialsParam.typeFullName shouldBe "Test0.ts::program:Test:run:<anon-class>0"
// should not produce dangling nodes that are meant to be inside procedures
cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0
cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0
cpg.identifier.count(_.refsTo.size > 1) shouldBe 0
cpg.identifier.whereNot(_.refsTo).size shouldBe 0
// should not produce assignment calls directly under typedecls
Expand All @@ -362,7 +362,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") {
val List(credentialsParam) = cpg.parameter.nameExact("param1_0").l
credentialsParam.typeFullName shouldBe "Test0.ts::program:apiCall:<anon-class>0"
// should not produce dangling nodes that are meant to be inside procedures
cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0
cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0
cpg.identifier.count(_.refsTo.size > 1) shouldBe 0
cpg.identifier.whereNot(_.refsTo).size shouldBe 0
// should not produce assignment calls directly under typedecls
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) {
//
private def astForForWithDestructuringLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = {
val loopRangeText = expr.getLoopRange.getText
val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}"
val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}"
val localForIterator = localNode(expr, iteratorName, iteratorName, TypeConstants.any)
val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any)
val iteratorLocalAst = Ast(localForIterator).withRefEdge(iteratorAssignmentLhs, localForIterator)
Expand Down Expand Up @@ -182,7 +182,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) {
//
private def astForForWithSimpleVarLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = {
val loopRangeText = expr.getLoopRange.getText
val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}"
val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}"
val iteratorLocal = localNode(expr, iteratorName, iteratorName, TypeConstants.any)
val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any)
val iteratorLocalAst = Ast(iteratorLocal).withRefEdge(iteratorAssignmentLhs, iteratorLocal)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture
import io.shiftleft.codepropertygraph.generated.Operators
import io.shiftleft.codepropertygraph.generated.edges.Argument
import io.shiftleft.semanticcpg.language.*
import overflowdb.traversal.jIteratortoTraversal

class ComplexExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) {
"CPG for code with _and_/_or_ operator and try-catch as one of the arguments" should {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.MethodRef
import io.shiftleft.codepropertygraph.generated.nodes.Return
import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl
import io.shiftleft.semanticcpg.language.*
import overflowdb.traversal.jIteratortoTraversal

class LambdaTests extends KotlinCode2CpgFixture(withOssDataflow = false, withDefaultJars = true) {
"CPG for code with a simple lambda which captures a method parameter" should {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package io.joern.php2cpg.passes

import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.PropertyNames
import io.shiftleft.codepropertygraph.generated.{Cpg, Properties, PropertyNames}
import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, NamespaceBlock, Method, TypeDecl}
import io.shiftleft.passes.ForkJoinParallelCpgPass
import io.shiftleft.semanticcpg.language.*
Expand All @@ -15,7 +14,7 @@ class AstParentInfoPass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg)
override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = {
findParent(node).foreach { parentNode =>
val astParentType = parentNode.label
val astParentFullName = parentNode.property(PropertyNames.FULL_NAME)
val astParentFullName = parentNode.property(Properties.FullName)

diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_TYPE, astParentType)
diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_FULL_NAME, astParentFullName)
Expand Down
Loading
Loading