Merge pull request #45 from Ostorlab/fix/whatweb_http

Fix whatweb scanning non HTTP services.

agent/whatweb_agent.py

@@ -128,7 +128,7 @@ class IPTarget(BaseTarget):
     def target(self) -> str:
         """Prepare target."""
         url = ""
-        if self.schema is not None:
+        if self.schema is not None and self.schema in ("https", "http"):
             url += f"{self.schema}://"
 
         url += self.name
@@ -163,16 +163,18 @@ def process(self, message: msg.Message) -> None:
         """
         logger.info("processing message of selector : %s", message.selector)
         targets = self._prepare_targets(message)
+        logger.info("Generated targets %s", targets)
         if self._should_target_be_processed(message) is False:
             return
 
         for target in targets:
             try:
+                logger.info("Scanning target %s", target)
                 with tempfile.NamedTemporaryFile() as fp:
                     self._start_scan(target, fp.name)
                     self._parse_emit_result(target, io.BytesIO(fp.read()))
             except subprocess.CalledProcessError as e:
-                logger.error(e)
+                logger.error("Error scanning target `%s`: %s", target, e)
 
     def _prepare_targets(self, message: msg.Message) -> List[IPTarget | DomainTarget]:
         """Returns a list of target objects to be scanned."""

tests/conftest.py

@@ -75,10 +75,16 @@ def ip_msg_with_port_schema_mask() -> m.Message:
 
 
 @pytest.fixture
-def ip_msg_with_port_schema_mask_2() -> m.Message:
+def ip_tcp_message() -> m.Message:
     """Creates a dummy message of type v3.asset.ip.v4.port.service for testing purposes."""
     input_selector = "v3.asset.ip.v4.port.service"
-    input_data = {"host": "192.168.0.0", "port": 80, "mask": "32", "protocol": "http"}
+    input_data = {
+        "host": "192.168.0.0",
+        "port": 80,
+        "mask": "32",
+        "protocol": "tcp",
+        "version": 4,
+    }
     message = m.Message.from_data(selector=input_selector, data=input_data)
     return message
 

tests/whatweb_test.py

@@ -569,3 +569,25 @@ def testWhatWebAgent_whenIPAssetHasIncorrectVersion_raiseValueError(
     """Test the CIDR Limit in case IP has incorrect version."""
     with pytest.raises(ValueError, match="Incorrect ip version 5."):
         test_agent.process(scan_message_ipv_with_incorrect_version)
+
+
+def testWhatWebAgent_whenSchemeIsNotHTTP_defaultToNoScheme(
+    agent_mock: List[message.Message],
+    whatweb_test_agent: whatweb_agent.AgentWhatWeb,
+    ip_tcp_message: message.Message,
+    mocker: plugin.MockerFixture,
+) -> None:
+    run_mock = mocker.patch("subprocess.run", return_value=None)
+    with tempfile.TemporaryFile() as fp:
+        mocker.patch("tempfile.NamedTemporaryFile", return_value=fp)
+        with open(f"{pathlib.Path(__file__).parent}/ip_output.json", "rb") as op:
+            fp.write(op.read())
+            fp.seek(0)
+            whatweb_test_agent.process(ip_tcp_message)
+
+            assert run_mock.call_count == 1
+            assert run_mock.call_args[0][0] == [
+                "./whatweb",
+                "--log-json-verbose=11",
+                "192.168.0.0:80",
+            ]