Merge pull request #19 from Ostorlab/fix/increase_semgrep_sizelimit_a…

…nd_timout

Increase file size and timeout values for semgrep command

agent/semgrep_agent.py

@@ -22,7 +22,13 @@
 )
 logger = logging.getLogger(__name__)
 
-COMMAND_TIMEOUT = 90
+COMMAND_TIMEOUT = 120
+# Number of semgrep rules that can time out on a file before the file is skipped, 0 will have no limit.
+TIMEOUT_THRESHOLD = 0
+# 500MB
+FILE_SIZE_LIMIT = 500 * 1024 * 1024
+# 2GB
+DEFAULT_MEMORY_LIMIT = 2 * 1024 * 1024 * 1024
 
 FILE_TYPE_BLACKLIST = (
     ".car",
@@ -51,8 +57,25 @@
 )
 
 
-def _run_analysis(input_file_path: str) -> tuple[bytes, bytes] | None:
-    command = ["semgrep", "-q", "--config", "auto", "--json", input_file_path]
+def _run_analysis(
+    input_file_path: str, max_memory_limit: int = DEFAULT_MEMORY_LIMIT
+) -> tuple[bytes, bytes] | None:
+    command = [
+        "semgrep",
+        "-q",
+        "--config",
+        "auto",
+        "--timeout",
+        str(COMMAND_TIMEOUT),
+        "--timeout-threshold",
+        str(TIMEOUT_THRESHOLD),
+        "--max-target-bytes",
+        str(FILE_SIZE_LIMIT),
+        "--max-memory",
+        str(max_memory_limit),
+        "--json",
+        input_file_path,
+    ]
     try:
         output = subprocess.run(
             command, capture_output=True, check=True, timeout=COMMAND_TIMEOUT
@@ -81,6 +104,9 @@ def process(self, message: m.Message) -> None:
         """
         content = utils.get_file_content(message)
         path = message.data.get("path")
+        memory_limit = (
+            self.args.get("memory_limit", DEFAULT_MEMORY_LIMIT) or DEFAULT_MEMORY_LIMIT
+        )
 
         if content is None:
             logger.error("Received empty file.")
@@ -104,7 +130,7 @@ def process(self, message: m.Message) -> None:
                 infile.write(content)
             infile.flush()
 
-            output = _run_analysis(infile.name)
+            output = _run_analysis(infile.name, memory_limit)
 
             if output is None:
                 logger.error("Subprocess completed with errors.")

ostorlab.yaml

@@ -53,3 +53,7 @@ out_selectors:
   - v3.report.vulnerability
 docker_file_path : Dockerfile 
 docker_build_root : .
+args:
+  - name: "memory_limit"
+    description: "Memory limit for semgrep to use on a single file."
+    type: "number"

tests/semgrep_agent_test.py

@@ -341,3 +341,31 @@ def testAgentSemgrep_whenAnalysisRunsOnJsFile_emitsBackVulnerability(
     test_agent.process(scan_message_js_file)
 
     assert len(agent_mock) > 0
+
+
+def testAgentSemgrep_whenValidMessage_constructCorrectCommand(
+    test_agent: semgrep_agent.SemgrepAgent,
+    scan_message_file: message.Message,
+    mocker: plugin.MockerFixture,
+) -> None:
+    """Unit test testing semgrep command construction."""
+    command_mock = mocker.patch(
+        "subprocess.run",
+        side_effect=subprocess.CalledProcessError(cmd="", returncode=2),
+    )
+
+    test_agent.process(scan_message_file)
+
+    assert command_mock.call_args.args[0][0] == "semgrep"
+    assert command_mock.call_args.args[0][1] == "-q"
+    assert command_mock.call_args.args[0][2] == "--config"
+    assert command_mock.call_args.args[0][3] == "auto"
+    assert command_mock.call_args.args[0][4] == "--timeout"
+    assert command_mock.call_args.args[0][5] == "120"
+    assert command_mock.call_args.args[0][6] == "--timeout-threshold"
+    assert command_mock.call_args.args[0][7] == "0"
+    assert command_mock.call_args.args[0][8] == "--max-target-bytes"
+    assert command_mock.call_args.args[0][9] == "524288000"
+    assert command_mock.call_args.args[0][10] == "--max-memory"
+    assert command_mock.call_args.args[0][11] == "2147483648"
+    assert command_mock.call_args.args[0][12] == "--json"