Optimize Concurrency with WaitGroup

[ksg97031]

Analyzers

• Spring
• Django
• FastAPI
• Flask
• Kotlin

Reference

#491

[hahwul]

@ksg97031
케이스 바이 케이스긴 한데, 대상 소스코드 경로의 파일이 적은 케이스 등 일부 상황에서는 그냥 파일 루프 도는 것보다 병렬 처리가 느린 경우도 있었습니다. 그래서 분기가 필요할지 고민이긴 하네요. 일단 참고 차 공유드립니다 😊

[ksg97031]

@hahwul
그러네요 ㅠ 스프링하나 테스트해봤는데 속도가 비슷하게 나오네요.
명확히 속도 개선 나올 때까지 보류하겠습니다!

벤치마크

- 현재 코드
Benchmark 1: ./bin/noir -b ../mall/
  Time (mean ± σ):      1.541 s ±  0.006 s    [User: 1.512 s, System: 0.023 s]
  Range (min … max):    1.533 s …  1.551 s    10 runs
 
- WaitGroup 적용
Benchmark 1: ./bin/noir -b ../mall/ --concurrency 50
  Time (mean ± σ):      1.558 s ±  0.011 s    [User: 1.529 s, System: 0.023 s]
  Range (min … max):    1.542 s …  1.570 s    10 runs

작업코드

diff --git a/src/analyzer/analyzers/java/spring.cr b/src/analyzer/analyzers/java/spring.cr
index f96d02c..7d111da 100644
--- a/src/analyzer/analyzers/java/spring.cr
+++ b/src/analyzer/analyzers/java/spring.cr
@@ -11,259 +11,251 @@ module Analyzer::Java
     def analyze
       parser_map = Hash(String, JavaParser).new
       package_map = Hash(String, Hash(String, ClassModel)).new
-      webflux_base_path_map = Hash(String, String).new
+      webflux_base_path_map = parse_config_files
 
-      Dir.glob("#{@base_path}/**/*") do |path|
-        url = ""
-
-        # Extract the Webflux base path from 'application.yml' in specified directories
-        if File.directory?(path)
-          if path.ends_with?("/src")
-            application_yml_path = File.join(path, "main/resources/application.yml")
-            if File.exists?(application_yml_path)
-              begin
-                config = YAML.parse(File.read(application_yml_path))
-                spring = config["spring"]
-                webflux = spring["webflux"]
-                webflux_base_path = webflux["base-path"]
-
-                if webflux_base_path
-                  webflux_base_path_map[path] = webflux_base_path.as_s
-                end
-              rescue e
-                # Handle parsing errors if necessary
-              end
-            end
+      # Source Analysis
+      channel = Channel(String).new
 
-            application_properties_path = File.join(path, "main/resources/application.properties")
-            if File.exists?(application_properties_path)
-              begin
-                properties = File.read(application_properties_path)
-                base_path = properties.match(/spring\.webflux\.base-path\s*=\s*(.*)/)
-                if base_path
-                  webflux_base_path = base_path[1]
-                  webflux_base_path_map[path] = webflux_base_path if webflux_base_path
-                end
-              rescue e
-                # Handle parsing errors if necessary
-              end
-            end
+      begin
+        spawn do
+          Dir.glob("#{@base_path}/**/*") do |path|
+            next if File.directory?(path)
+            channel.send(path)
           end
-        elsif File.exists?(path) && path.ends_with?(".java")
-          webflux_base_path = find_base_path(path, webflux_base_path_map)
-          # Load Java file content into cache for processing
-          content = File.read(path, encoding: "utf-8", invalid: :skip)
-          FILE_CONTENT_CACHE[path] = content
-
-          # Process files that include Spring MVC bindings for routing
-          spring_web_bind_package = "org.springframework.web.bind.annotation."
-          has_spring_bindings = content.includes?(spring_web_bind_package)
-          if has_spring_bindings
-            if parser_map.has_key?(path)
-              parser = parser_map[path]
-              tokens = parser.tokens
-            else
-              parser = create_parser(Path.new(path), content)
-              tokens = parser.tokens
-              parser_map[path] = parser
-            end
-
-            package_name = parser.get_package_name(tokens)
-            next if package_name == ""
-            root_source_directory : Path = parser.get_root_source_directory(path, package_name)
-            package_directory = Path.new(path).dirname
-
-            # Import packages
-            import_map = Hash(String, ClassModel).new
-            parser.import_statements.each do |import_statement|
-              import_path = import_statement.gsub(".", "/")
-              if import_path.ends_with?("/*")
-                import_directory = root_source_directory.join(import_path[..-3])
-                if Dir.exists?(import_directory)
-                  Dir.glob("#{import_directory}/*.java") do |_path|
-                    next if path == _path
-                    if !parser_map.has_key?(_path)
-                      _parser = create_parser(Path.new(_path))
-                      parser_map[_path] = _parser
-                    else
-                      _parser = parser_map[_path]
-                    end
-
-                    _parser.classes.each do |package_class|
-                      import_map[package_class.name] = package_class
-                    end
-                  end
-                end
-              else
-                source_path = root_source_directory.join(import_path + ".java")
-                next if source_path.dirname == package_directory || !File.exists?(source_path)
-                if !parser_map.has_key?(source_path.to_s)
-                  _parser = create_parser(source_path)
-                  parser_map[source_path.to_s] = _parser
-                  _parser.classes.each do |package_class|
-                    import_map[package_class.name] = package_class
-                  end
-                else
-                  _parser = parser_map[source_path.to_s]
-                  _parser.classes.each do |package_class|
-                    import_map[package_class.name] = package_class
-                  end
-                end
-              end
-            end
-
-            # Import packages from the same directory
-            package_class_map = package_map[package_directory]?
-            if package_class_map.nil?
-              package_class_map = Hash(String, ClassModel).new
-              Dir.glob("#{package_directory}/*.java") do |_path|
-                next if path == _path
-                if !parser_map.has_key?(_path)
-                  _parser = create_parser(Path.new(_path))
-                  parser_map[_path] = _parser
-                else
-                  _parser = parser_map[_path]
-                end
-
-                _parser.classes.each do |package_class|
-                  package_class_map[package_class.name] = package_class
-                end
-
-                parser.classes.each do |package_class|
-                  package_class_map[package_class.name] = package_class
-                end
+          channel.close
+        end
 
-                package_map[package_directory] = package_class_map
-              end
-            end
+        WaitGroup.wait do |wg|
+          @options["concurrency"].to_s.to_i.times do
+            wg.spawn do
+              loop do
+                begin
+                  path = channel.receive?
+                  break if path.nil?
+                  url = ""
+
+                  if File.exists?(path) && path.ends_with?(".java")
+                    webflux_base_path = find_base_path(path, webflux_base_path_map)
+                    # Load Java file content into cache for processing
+                    content = File.read(path, encoding: "utf-8", invalid: :skip)
+                    FILE_CONTENT_CACHE[path] = content
+
+                    # Process files that include Spring MVC bindings for routing
+                    spring_web_bind_package = "org.springframework.web.bind.annotation."
+                    has_spring_bindings = content.includes?(spring_web_bind_package)
+                    if has_spring_bindings
+                      if parser_map.has_key?(path)
+                        parser = parser_map[path]
+                        tokens = parser.tokens
+                      else
+                        parser = create_parser(Path.new(path), content)
+                        tokens = parser.tokens
+                        parser_map[path] = parser
+                      end
 
-            # Extract URL mappings and methods from Spring MVC annotated classes
-            class_map = package_class_map.merge(import_map)
-            parser.classes.each do |class_model|
-              class_annotation = class_model.annotations["RequestMapping"]?
-              if !class_annotation.nil?
-                next if class_annotation.params.size == 0
-                if class_annotation.params[0].size > 0
-                  class_path_token = class_annotation.params[0][-1]
-                  if class_path_token.type == :STRING_LITERAL
-                    url = class_path_token.value[1..-2]
-                    if url.ends_with? "*"
-                      url = url[0..-2]
-                    end
-                  end
-                end
-              end
+                      package_name = parser.get_package_name(tokens)
+                      next if package_name == ""
+                      root_source_directory : Path = parser.get_root_source_directory(path, package_name)
+                      package_directory = Path.new(path).dirname
+
+                      # Import packages
+                      import_map = Hash(String, ClassModel).new
+                      parser.import_statements.each do |import_statement|
+                        import_path = import_statement.gsub(".", "/")
+                        if import_path.ends_with?("/*")
+                          import_directory = root_source_directory.join(import_path[..-3])
+                          if Dir.exists?(import_directory)
+                            Dir.glob("#{import_directory}/*.java") do |_path|
+                              next if path == _path
+                              if !parser_map.has_key?(_path)
+                                _parser = create_parser(Path.new(_path))
+                                parser_map[_path] = _parser
+                              else
+                                _parser = parser_map[_path]
+                              end
 
-              class_model.methods.values.each do |method|
-                method.annotations.values.each do |method_annotation|
-                  url_paths = Array(String).new
-
-                  # Spring MVC method mappings
-                  request_methods = Array(String).new
-                  if method_annotation.name.ends_with? "Mapping"
-                    parameter_format = nil
-                    annotation_parameters = method_annotation.params
-                    annotation_parameters.each do |annotation_parameter_tokens|
-                      if annotation_parameter_tokens.size > 2
-                        annotation_parameter_key = annotation_parameter_tokens[0].value
-                        annotation_parameter_value = annotation_parameter_tokens[-1].value
-                        if annotation_parameter_key == "method"
-                          if ["}", "]"].includes?(annotation_parameter_value)
-                            # Handle methods declared with multiple HTTP verbs
-                            annotation_parameter_tokens.reverse_each do |token|
-                              break if token.value == "method"
-                              next if [:LBRACE, :RBRACE, :LBRACK, :RBRACK, :COMMA, :DOT].includes?(token.type)
-                              http_methods = ["GET", "POST", "PUT", "DELETE", "PATCH"]
-                              if http_methods.includes?(token.value)
-                                request_methods.push(token.value)
+                              _parser.classes.each do |package_class|
+                                import_map[package_class.name] = package_class
                               end
                             end
-                          else
-                            request_methods.push(annotation_parameter_value)
                           end
-                        elsif annotation_parameter_key == "consumes"
-                          # Set parameter format based on the 'consumes' attribute of the annotation.
-                          if annotation_parameter_value.ends_with? "APPLICATION_FORM_URLENCODED_VALUE"
-                            parameter_format = "form"
-                          elsif annotation_parameter_value.ends_with? "APPLICATION_JSON_VALUE"
-                            parameter_format = "json"
+                        else
+                          source_path = root_source_directory.join(import_path + ".java")
+                          next if source_path.dirname == package_directory || !File.exists?(source_path)
+                          if !parser_map.has_key?(source_path.to_s)
+                            _parser = create_parser(source_path)
+                            parser_map[source_path.to_s] = _parser
+                            _parser.classes.each do |package_class|
+                              import_map[package_class.name] = package_class
+                            end
+                          else
+                            _parser = parser_map[source_path.to_s]
+                            _parser.classes.each do |package_class|
+                              import_map[package_class.name] = package_class
+                            end
                           end
                         end
                       end
-                    end
 
-                    if webflux_base_path.ends_with?("/") && url.starts_with?("/")
-                      webflux_base_path = webflux_base_path[..-2]
-                    end
+                      # Import packages from the same directory
+                      package_class_map = package_map[package_directory]?
+                      if package_class_map.nil?
+                        package_class_map = Hash(String, ClassModel).new
+                        Dir.glob("#{package_directory}/*.java") do |_path|
+                          next if path == _path
+                          if !parser_map.has_key?(_path)
+                            _parser = create_parser(Path.new(_path))
+                            parser_map[_path] = _parser
+                          else
+                            _parser = parser_map[_path]
+                          end
 
-                    # Parse and construct endpoints for methods annotated with 'RequestMapping' or specific HTTP methods
-                    if method_annotation.name == "RequestMapping"
-                      url_paths = [""]
-                      if method_annotation.params.size > 0
-                        url_paths = get_mapping_path(parser, tokens, method_annotation.params)
-                      end
+                          _parser.classes.each do |package_class|
+                            package_class_map[package_class.name] = package_class
+                          end
 
-                      line = method_annotation.tokens[0].line
-                      details = Details.new(PathInfo.new(path, line))
+                          parser.classes.each do |package_class|
+                            package_class_map[package_class.name] = package_class
+                          end
 
-                      if request_methods.empty?
-                        # Handle default HTTP methods if no specific method is annotated
-                        ["GET", "POST", "PUT", "DELETE", "PATCH"].each do |_request_method|
-                          parameters = get_endpoint_parameters(parser, _request_method, method, parameter_format, class_map)
-                          url_paths.each do |url_path|
-                            @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", _request_method, parameters, details)
+                          package_map[package_directory] = package_class_map
+                        end
+                      end
+
+                      # Extract URL mappings and methods from Spring MVC annotated classes
+                      class_map = package_class_map.merge(import_map)
+                      parser.classes.each do |class_model|
+                        class_annotation = class_model.annotations["RequestMapping"]?
+                        if !class_annotation.nil?
+                          next if class_annotation.params.size == 0
+                          if class_annotation.params[0].size > 0
+                            class_path_token = class_annotation.params[0][-1]
+                            if class_path_token.type == :STRING_LITERAL
+                              url = class_path_token.value[1..-2]
+                              if url.ends_with? "*"
+                                url = url[0..-2]
+                              end
+                            end
                           end
                         end
-                      else
-                        # Create endpoints for annotated HTTP methods
-                        url_paths.each do |url_path|
-                          request_methods.each do |request_method|
-                            parameters = get_endpoint_parameters(parser, request_method, method, parameter_format, class_map)
-                            @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", request_method, parameters, details)
+
+                        class_model.methods.values.each do |method|
+                          method.annotations.values.each do |method_annotation|
+                            url_paths = Array(String).new
+
+                            # Spring MVC method mappings
+                            request_methods = Array(String).new
+                            if method_annotation.name.ends_with? "Mapping"
+                              parameter_format = nil
+                              annotation_parameters = method_annotation.params
+                              annotation_parameters.each do |annotation_parameter_tokens|
+                                if annotation_parameter_tokens.size > 2
+                                  annotation_parameter_key = annotation_parameter_tokens[0].value
+                                  annotation_parameter_value = annotation_parameter_tokens[-1].value
+                                  if annotation_parameter_key == "method"
+                                    if ["}", "]"].includes?(annotation_parameter_value)
+                                      # Handle methods declared with multiple HTTP verbs
+                                      annotation_parameter_tokens.reverse_each do |token|
+                                        break if token.value == "method"
+                                        next if [:LBRACE, :RBRACE, :LBRACK, :RBRACK, :COMMA, :DOT].includes?(token.type)
+                                        http_methods = ["GET", "POST", "PUT", "DELETE", "PATCH"]
+                                        if http_methods.includes?(token.value)
+                                          request_methods.push(token.value)
+                                        end
+                                      end
+                                    else
+                                      request_methods.push(annotation_parameter_value)
+                                    end
+                                  elsif annotation_parameter_key == "consumes"
+                                    # Set parameter format based on the 'consumes' attribute of the annotation.
+                                    if annotation_parameter_value.ends_with? "APPLICATION_FORM_URLENCODED_VALUE"
+                                      parameter_format = "form"
+                                    elsif annotation_parameter_value.ends_with? "APPLICATION_JSON_VALUE"
+                                      parameter_format = "json"
+                                    end
+                                  end
+                                end
+                              end
+
+                              if webflux_base_path.ends_with?("/") && url.starts_with?("/")
+                                webflux_base_path = webflux_base_path[..-2]
+                              end
+
+                              # Parse and construct endpoints for methods annotated with 'RequestMapping' or specific HTTP methods
+                              if method_annotation.name == "RequestMapping"
+                                url_paths = [""]
+                                if method_annotation.params.size > 0
+                                  url_paths = get_mapping_path(parser, tokens, method_annotation.params)
+                                end
+
+                                line = method_annotation.tokens[0].line
+                                details = Details.new(PathInfo.new(path, line))
+
+                                if request_methods.empty?
+                                  # Handle default HTTP methods if no specific method is annotated
+                                  ["GET", "POST", "PUT", "DELETE", "PATCH"].each do |_request_method|
+                                    parameters = get_endpoint_parameters(parser, _request_method, method, parameter_format, class_map)
+                                    url_paths.each do |url_path|
+                                      @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", _request_method, parameters, details)
+                                    end
+                                  end
+                                else
+                                  # Create endpoints for annotated HTTP methods
+                                  url_paths.each do |url_path|
+                                    request_methods.each do |request_method|
+                                      parameters = get_endpoint_parameters(parser, request_method, method, parameter_format, class_map)
+                                      @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", request_method, parameters, details)
+                                    end
+                                  end
+                                end
+                                break
+                              else
+                                # Handle other specific mapping annotations like 'GetMapping', 'PostMapping', etc
+                                mapping_annotations = ["GetMapping", "PostMapping", "PutMapping", "DeleteMapping", "PatchMapping"]
+                                mapping_index = mapping_annotations.index(method_annotation.name)
+                                if !mapping_index.nil?
+                                  line = method_annotation.tokens[0].line
+                                  request_method = mapping_annotations[mapping_index][0..-8].upcase
+                                  if parameter_format.nil? && request_method == "POST"
+                                    parameter_format = "form"
+                                  end
+                                  parameters = get_endpoint_parameters(parser, request_method, method, parameter_format, class_map)
+
+                                  url_paths = [""]
+                                  if method_annotation.params.size > 0
+                                    url_paths = get_mapping_path(parser, tokens, method_annotation.params)
+                                  end
+
+                                  details = Details.new(PathInfo.new(path, line))
+                                  url_paths.each do |url_path|
+                                    @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", request_method, parameters, details)
+                                  end
+                                  break
+                                end
+                              end
+                            end
                           end
                         end
                       end
-                      break
                     else
-                      # Handle other specific mapping annotations like 'GetMapping', 'PostMapping', etc
-                      mapping_annotations = ["GetMapping", "PostMapping", "PutMapping", "DeleteMapping", "PatchMapping"]
-                      mapping_index = mapping_annotations.index(method_annotation.name)
-                      if !mapping_index.nil?
-                        line = method_annotation.tokens[0].line
-                        request_method = mapping_annotations[mapping_index][0..-8].upcase
-                        if parameter_format.nil? && request_method == "POST"
-                          parameter_format = "form"
-                        end
-                        parameters = get_endpoint_parameters(parser, request_method, method, parameter_format, class_map)
-
-                        url_paths = [""]
-                        if method_annotation.params.size > 0
-                          url_paths = get_mapping_path(parser, tokens, method_annotation.params)
-                        end
-
-                        details = Details.new(PathInfo.new(path, line))
-                        url_paths.each do |url_path|
-                          @result << Endpoint.new("#{webflux_base_path}#{url}#{url_path}", request_method, parameters, details)
+                      # Extract and construct endpoints from reactive route configurations
+                      content.scan(REGEX_ROUTER_CODE_BLOCK) do |route_code|
+                        method_code = route_code[0]
+                        method_code.scan(REGEX_ROUTE_CODE_LINE) do |match|
+                          next if match.size != 4
+                          method = match[2]
+                          endpoint = match[3].gsub(/\n/, "")
+                          details = Details.new(PathInfo.new(path))
+                          @result << Endpoint.new("#{url}#{endpoint}", method, details)
                         end
-                        break
                       end
                     end
                   end
+                rescue e : File::NotFoundError
+                  logger.debug "File not found: #{path}"
                 end
               end
             end
-          else
-            # Extract and construct endpoints from reactive route configurations
-            content.scan(REGEX_ROUTER_CODE_BLOCK) do |route_code|
-              method_code = route_code[0]
-              method_code.scan(REGEX_ROUTE_CODE_LINE) do |match|
-                next if match.size != 4
-                method = match[2]
-                endpoint = match[3].gsub(/\n/, "")
-                details = Details.new(PathInfo.new(path))
-                @result << Endpoint.new("#{url}#{endpoint}", method, details)
-              end
-            end
           end
         end
       end
@@ -272,6 +264,46 @@ module Analyzer::Java
       @result
     end
 
+    def parse_config_files
+      webflux_base_path_map = Hash(String, String).new
+
+      Dir.glob("#{@base_path}/**/resources/") do |resources_path|
+        # Extract the Webflux base path from 'application.yml' and 'application.properties' in specified directories
+        application_yml_path = File.join(resources_path, "application.yml")
+        application_properties_path = File.join(resources_path, "application.properties")
+
+        if File.exists?(application_yml_path)
+          begin
+            config = YAML.parse(File.read(application_yml_path))
+            spring = config["spring"]
+            webflux = spring["webflux"]
+            webflux_base_path = webflux["base-path"]
+
+            if webflux_base_path
+              webflux_base_path_map[resources_path] = webflux_base_path.as_s
+            end
+          rescue e
+            logger.debug "Error parsing YAML: #{e.message}"
+          end
+        end
+
+        if File.exists?(application_properties_path)
+          begin
+            properties = File.read(application_properties_path)
+            base_path_match = properties.match(/spring\.webflux\.base-path\s*=\s*(.*)/)
+            if base_path_match
+              webflux_base_path = base_path_match[1]
+              webflux_base_path_map[resources_path] = webflux_base_path if webflux_base_path
+            end
+          rescue e
+            logger.debug "Error parsing properties: #{e.message}"
+          end
+        end
+      end
+
+      webflux_base_path_map
+    end
+
     def create_parser(path : Path, content : String = "")
       if content == ""
         if FILE_CONTENT_CACHE.has_key?(path.to_s)