Resolve scan failures in Python framework

src/analyzer/analyzers/python/django.cr

@@ -35,6 +35,7 @@ module Analyzer::Python
       # Find root Django URL configurations
       root_django_urls_list = find_root_django_urls()
       root_django_urls_list.each do |root_django_urls|
+        logger.debug "Found Django URL configurations in #{root_django_urls.filepath}"
         @django_base_path = root_django_urls.basepath
         extract_endpoints(root_django_urls).each do |endpoint|
           endpoints << endpoint
@@ -65,6 +66,7 @@ module Analyzer::Python
           spawn do
             begin
               next if File.directory?(file)
+              next if file.includes?("/site-packages/")
               if file.ends_with? ".py"
                 content = File.read(file, encoding: "utf-8", invalid: :skip)
                 content.scan(REGEX_ROOT_URLCONF) do |match|
@@ -79,7 +81,7 @@ module Analyzer::Python
                 end
               end
             rescue e : File::NotFoundError
-              @logger.debug "File not found: #{file}"
+              logger.debug "File not found: #{file}"
             end
           end
           Fiber.yield
@@ -93,6 +95,7 @@ module Analyzer::Python
 
     # Extract endpoints from a Django URL configuration file
     def extract_endpoints(django_urls : DjangoUrls) : Array(Endpoint)
+      logger.debug "Extracting endpoints from #{django_urls.filepath}"
       endpoints = [] of Endpoint
       url_base_path = File.dirname(django_urls.filepath)
 
@@ -126,9 +129,11 @@ module Analyzer::Python
           if File.exists?(new_route_path)
             new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath)
             details = Details.new(PathInfo.new(new_route_path))
-            extract_endpoints(new_django_urls).each do |endpoint|
-              endpoint.details = details
-              endpoints << endpoint
+            if new_django_urls.filepath != django_urls.filepath
+              extract_endpoints(new_django_urls).each do |endpoint|
+                endpoint.details = details
+                endpoints << endpoint
+              end
             end
           end
         end
@@ -171,6 +176,8 @@ module Analyzer::Python
 
     # Extract endpoints from a given file
     def extract_endpoints_from_file(url : ::String, filepath : ::String, function_or_class_name : ::String)
+      @logger.debug "Extracting endpoints from #{filepath}"
+
       endpoints = Array(Endpoint).new
       suspicious_http_methods = ["GET"]
       suspicious_params = Array(Param).new

src/analyzer/analyzers/python/fastapi.cr

@@ -13,6 +13,7 @@ module Analyzer::Python
         # Iterate through all Python files in the base path
         Dir.glob("#{base_path}/**/*.py") do |path|
           next if File.directory?(path)
+          next if path.includes?("/site-packages/")
           source = File.read(path, encoding: "utf-8", invalid: :skip)
 
           source.each_line do |line|

src/analyzer/analyzers/python/flask.cr

@@ -40,6 +40,9 @@ module Analyzer::Python
       # Iterate through all Python files in the base path
       Dir.glob("#{base_path}/**/*.py") do |path|
         next if File.directory?(path)
+        next if path.includes?("/site-packages/")
+        @logger.debug "Analyzing #{path}"
+
         File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
           lines = file.each_line.to_a
           next unless lines.any?(&.includes?("flask"))
@@ -274,8 +277,12 @@ module Analyzer::Python
     def create_parser(path : ::String, content : ::String = "") : PythonParser
       content = fetch_file_content(path) if content.empty?
       lexer = PythonLexer.new
+      @logger.debug "Tokenizing #{path}"
       tokens = lexer.tokenize(content)
-      PythonParser.new(path, tokens, @parsers)
+      @logger.debug "Parsing #{path}"
+      parser = PythonParser.new(path, tokens, @parsers)
+      @logger.debug "Parsed #{path}"
+      parser
     end
 
     # Get a parser for a given path

src/miniparsers/python.cr

@@ -136,13 +136,6 @@ class PythonParser
     end
 
     import_statements.each do |import_statement|
-      @debug = false
-      if import_statement.size == 2
-        if import_statement[0] == "." && import_statement[1] == "models"
-          @debug = true
-        end
-      end
-
       name = import_statement[-1]
 
       # Check if the name has an alias
@@ -160,11 +153,13 @@ class PythonParser
         package_dir = File.dirname(@path)
         import_statement.shift
       end
-      if import_statement.size == 2 && import_statement[0] == "models"
-        @debug = true
-      end
+
       import_statement.each_with_index do |import_part, _index|
         path = File.join(package_dir, import_part)
+        if import_part == ".."
+          path = File.dirname(package_dir)
+        end
+
         # Order of checking is important
         if File.directory?(path)
           package_dir = path