metlo-labs · AHarmlessPyro · Dec 4, 2022 · Dec 3, 2022 · Dec 3, 2022 · Dec 3, 2022
diff --git a/cli/package.json b/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@metlo/cli",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

diff --git a/cli/src/gcp/setup.ts b/cli/src/gcp/setup.ts
@@ -337,11 +337,16 @@ const create_mig = async (
         network: network_url,
         subnet: destination_subnetwork_url,
         imageTemplateName: imageTemplateName,
-        startupScript: `#!/bin/bash
-        echo "METLO_ADDR=${machineInfoResp['_url']}" >> /opt/metlo/credentials
-        echo "METLO_KEY=${machineInfoResp['_apiKey']}" >>  /opt/metlo/credentials        
-        sudo systemctl enable metlo-ingestor.service
-        sudo systemctl start metlo-ingestor.service`
+        startupScript: `#!/bin/bash -e
+        sudo mkdir -p /opt/metlo
+        sudo touch /opt/metlo/credentials
+        export user_exists=$(getent passwd suricata)
+        [ ! -z $user_exists ] || echo "METLO_ADDR=${machineInfoResp['_url']}" >> /opt/metlo/credentials
+        [ ! -z $user_exists ] || echo "METLO_KEY=${machineInfoResp['_apiKey']}" >>  /opt/metlo/credentials
+        [ ! -z $user_exists ] || sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/install.sh
+        [ ! -z $user_exists ] || sudo chmod +x install.sh
+        [ ! -z $user_exists ] || sudo -E ./install.sh
+        `
     })
     let img_resp = await wait_for_global_operation(
         image_resp[0].latestResponse.name,
@@ -581,7 +586,7 @@ const resolveImageURL = (zone) => {
     }
 }
 
-export const gcpTrafficMirrorSetup = async ({ force }) => {    
+export const gcpTrafficMirrorSetup = async ({ force }) => {
     const id = uuidv4()
     const data = {}
     try {
@@ -614,7 +619,8 @@ export const gcpTrafficMirrorSetup = async ({ force }) => {
             data["firewallOutboundRuleUrl"] = firewallOutboundRuleUrl
             const { routerURL } = await createCloudRouter(conn, networkUrl, destinationSubnetworkUrl, id)
             data["routerURL"] = routerURL
-            const { imageTemplateUrl, instanceGroupName, instanceUrl } = await create_mig(conn, networkUrl, destinationSubnetworkUrl, resolveImageURL(zone), id)
+            const imageURL = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221202"
+            const { imageTemplateUrl, instanceGroupName, instanceUrl } = await create_mig(conn, networkUrl, destinationSubnetworkUrl, imageURL, id)
             data['imageTemplateUrl'] = imageTemplateUrl
             data['instanceGroupName'] = instanceGroupName
             data['instanceUrl'] = instanceUrl

diff --git a/deploy/cloudformation/metlo-ingestor-deployment.json b/deploy/cloudformation/metlo-ingestor-deployment.json
@@ -1,6 +1,6 @@
 {
     "AWSTemplateFormatVersion": "2010-09-09",
-    "Description": "Here are some details about the template.",
+    "Description": "Cloudformation Template for Metlo Ingestor.",
     "Metadata": {
         "AWS::CloudFormation::Designer": {
             "12d7e160-0a80-41e1-9076-14d9d321b425": {
@@ -95,9 +95,12 @@
                             "\n",
                             [
                                 "#!/bin/bash -xve",
-                                {
+                                "sudo mkdir -p /opt/metlo",
+                                "sudo touch /opt/metlo/credentials",
+                                "export user_exists=$(getent passwd suricata)",
+                               {
                                     "Fn::Sub": [
-                                        "echo 'METLO_ADDR=${METADDR}' >> opt/metlo/credentials",
+                                        "[ ! -z $user_exists ] || echo 'METLO_ADDR=${METADDR}' >> /opt/metlo/credentials",
                                         {
                                             "METADDR": {
                                                 "Ref": "metloAddress"
@@ -107,16 +110,17 @@
                                 },
                                 {
                                     "Fn::Sub": [
-                                        "echo 'METLO_KEY=${METKEY}' >> opt/metlo/credentials",
+                                        "[ ! -z $user_exists ] || echo 'METLO_KEY=${METKEY}' >> /opt/metlo/credentials",
                                         {
                                             "METKEY": {
                                                 "Ref": "metloKey"
                                             }
                                         }
                                     ]
                                 },
-                                "sudo systemctl enable metlo-ingestor.service",
-                                "sudo systemctl start metlo-ingestor.service",
+                                "[ ! -z $user_exists ] || sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/install.sh",
+                                "[ ! -z $user_exists ] || sudo chmod +x install.sh",
+                                "[ ! -z $user_exists ] || sudo -E ./install.sh",
                                 "sudo apt-get update -y",
                                 "sudo apt-get install -y python python-setuptools",
                                 "sudo mkdir -p /opt/aws/bin",
@@ -193,34 +197,88 @@
             "Type": "AWS::EC2::VPC::Id"
         },
         "metloAddress": {
-            "Description": "Host address where the Metlo Collector resides",
+            "Description": "Host address where the Metlo Collector resides (http[s]://<YOUR_METLO_HOST>:8081)",
             "Type": "String"
         },
         "metloKey": {
-            "Description": "API KEY for Metlo Collector",
+            "Description": "API KEY for Metlo Collector. You can generate an API Key on the settings page",
             "Type": "String"
         }
     },
     "Mappings": {
         "RegionMap": {
-            "us-east-1": {
-                "AMI": "ami-0314228ab1dfca22c"
+            "af-south-1": {
+                "AMI": "ami-062cdcd18683ee84e"
             },
-            "us-east-2": {
-                "AMI": "ami-000c7bdfd2cc748ee"
+            "ap-east-1": {
+                "AMI": "ami-0476827462b538638"
             },
-            "us-west-1": {
-                "AMI": "ami-092b652ebe2ef2353"
+            "ap-northeast-1": {
+                "AMI": "ami-05c9a233b97c2a78e"
             },
-            "us-west-2": {
-                "AMI": "ami-0f237f0669df21823"
+            "ap-northeast-2": {
+                "AMI": "ami-05f5974ae1e61d78e"
+            },
+            "ap-northeast-3": {
+                "AMI": "ami-04144e5c57e5820b0"
             },
             "ap-south-1": {
-                "AMI": "ami-0fa1f6b93c28a21e4"
+                "AMI": "ami-052639b6127cfb32d"
+            },
+            "ap-southeast-1": {
+                "AMI": "ami-03d4b39cdeb4b7339"
+            },
+            "ap-southeast-2": {
+                "AMI": "ami-0c9b6758e5d5a9558"
+            },
+            "ap-southeast-3": {
+                "AMI": "ami-04ff67f9643e6df11"
+            },
+            "ca-central-1": {
+                "AMI": "ami-0872f26db03af5118"
+            },
+            "eu-central-1": {
+                "AMI": "ami-00648c36e527032ec"
+            },
+            "eu-central-2": {
+                "AMI": "ami-0dae81a048b6cc0be"
+            },
+            "eu-north-1": {
+                "AMI": "ami-03d20f9dd906ec688"
+            },
+            "eu-south-1": {
+                "AMI": "ami-0d31fd2c8fa64baf0"
+            },
+            "eu-west-1": {
+                "AMI": "ami-0c1bebf6df0bafa23"
             },
             "eu-west-2": {
-                "AMI": "ami-0efb31db7e8ceb9a2"
+                "AMI": "ami-05bfd03d0709e3ecb"
+            },
+            "eu-west-3": {
+                "AMI": "ami-058cbb29e232feceb"
+            },
+            "me-central-1": {
+                "AMI": "ami-09bc076c1e48cc663"
+            },
+            "me-south-1": {
+                "AMI": "ami-0799b0ab527e14c62"
+            },
+            "sa-east-1": {
+                "AMI": "ami-06eb89b14ac4f4029"
+            },
+            "us-east-1": {
+                "AMI": "ami-072d6c9fae3253f26"
+            },
+            "us-east-2": {
+                "AMI": "ami-0cb81cb394fc2e305"
+            },
+            "us-west-1": {
+                "AMI": "ami-0ff4dd898acfb0e67"
+            },
+            "us-west-2": {
+                "AMI": "ami-0080e1081b2c5aef6"
             }
         }
     }
-}
+}
diff --git a/deploy/suricata/install.sh b/deploy/suricata/install.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+sudo useradd -m suricata
+sudo usermod -aG sudo suricata
+echo "suricata:suricata" | sudo chpasswd
+
+export WHOAMI=suricata
+
+sudo apt update -y
+sudo apt install wget -y
+
+cd /home/$WHOAMI
+mkdir -p /home/$WHOAMI/suricata
+
+sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/install_deps.sh -P /home/$WHOAMI/suricata
+sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/install_nvm.sh -P /home/$WHOAMI/suricata
+sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/local.rules -P /home/$WHOAMI/suricata
+sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/metlo-ingestor.service -P /home/$WHOAMI/suricata
+sudo wget https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/suricata/suricata.yaml -P /home/$WHOAMI/suricata
+
+chmod +x /home/$WHOAMI/suricata/install_nvm.sh
+chmod +x /home/$WHOAMI/suricata/install_deps.sh
+
+sudo WHOAMI=$WHOAMI -i -u $WHOAMI /home/$WHOAMI/suricata/install_nvm.sh
+sudo PATH=$PATH WHOAMI=$WHOAMI /home/$WHOAMI/suricata/install_deps.sh
diff --git a/deploy/suricata/install_deps.sh b/deploy/suricata/install_deps.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+echo "suricata" | sudo  apt update -y
+echo "suricata" | sudo  apt install software-properties-common -y
+
+HOME_PATH=/home/$WHOAMI
+
+export NVM_DIR=$HOME_PATH/.nvm;
+source $NVM_DIR/nvm.sh;
+
+
+echo "suricata" | sudo  apt-get install wget git -y
+
+echo "CREATING DIRECTORIES"
+echo "suricata" | sudo  mkdir -p /etc/suricata-logs
+echo "suricata" | sudo  mkdir -p /var/lib/suricata
+echo "suricata" | sudo  mkdir -p /var/lib/suricata/rules
+echo "suricata" | sudo  mkdir -p /usr/local/nvm
+echo "suricata" | sudo  mkdir -p /etc/metlo-ingestor
+
+echo "SETTING PERMISSIONS"
+echo "suricata" | sudo  chmod 777 /etc/suricata-logs
+echo "suricata" | sudo  chmod 777 /etc/metlo-ingestor
+
+echo "suricata" | sudo  rm -rf /etc/metlo-ingestor
+
+echo "INSTALLING SURICATA"
+echo "suricata" | sudo  add-apt-repository ppa:oisf/suricata-stable -y
+echo "suricata" | sudo  apt satisfy "suricata (<<7.0.0)" -y
+echo "suricata" | sudo  systemctl enable suricata.service
+echo "suricata" | sudo  systemctl stop suricata.service
+
+
+echo "Get network interface"
+echo "suricata" | sudo  cp /home/$WHOAMI/suricata/local.rules /var/lib/suricata/rules/local.rules -f
+INTERFACE=$(ip link | egrep "ens[0-9]*" -o)
+[ ! -z "$INTERFACE" ] || INTERFACE=$(ip link | egrep "[0-9]*" -o)
+echo "Placing packet capture on interface $INTERFACE"
+sed -i "s/%interface/$INTERFACE/" /home/$WHOAMI/suricata/suricata.yaml
+echo "suricata" | sudo  cp /home/$WHOAMI/suricata/suricata.yaml /etc/suricata/suricata.yaml -f
+
+
+echo "CLONING INGESTOR"
+git clone https://github.com/metlo-labs/metlo.git /etc/metlo-ingestor
+
+yarn --cwd /etc/metlo-ingestor/ingestors/suricata install
+yarn --cwd /etc/metlo-ingestor/ingestors/suricata build
+
+cd ~
+# Use ~ as separator since HOME can have escapable characters which will conflict with forward-slash
+# Replace home directory to properly set nvm directory
+sed -i "s~%home~$HOME_PATH~" /home/$WHOAMI/suricata/metlo-ingestor.service
+
+echo "ADDING SERVICE"
+echo "suricata" | sudo  mv /home/$WHOAMI/suricata/metlo-ingestor.service /lib/systemd/system/metlo-ingestor.service -f
+
+echo "STARTING SERVICES"
+echo "suricata" | sudo  systemctl daemon-reload
+echo "suricata" | sudo  systemctl enable metlo-ingestor.service
+echo "suricata" | sudo  systemctl start metlo-ingestor.service
+echo "suricata" | sudo  systemctl start suricata.service
diff --git a/deploy/suricata/install_nvm.sh b/deploy/suricata/install_nvm.sh
@@ -0,0 +1,7 @@
+#!/bin/bash -e
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
+echo "Installing node and yarn"
+source /home/$WHOAMI/.nvm/nvm.sh
+nvm install 17.9.1
+nvm use 17.9.1
+npm install -g yarn
diff --git a/deploy/suricata/local.rules b/deploy/suricata/local.rules
@@ -0,0 +1 @@
+alert http !169.254.169.254 any -> any any (msg:"TEST"; flow:established,to_client; http.response_body; pcre:/./; sid:1; rev:1; threshold: type limit, track by_rule, seconds 1, count 30;)
diff --git a/deploy/suricata/metlo-ingestor.service b/deploy/suricata/metlo-ingestor.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Metlo NodeJS ingestor application
+After=network-online.target
+
+[Service]
+Restart=on-failure
+WorkingDirectory=/etc/metlo-ingestor/ingestors/suricata/dist
+ExecStart=%home/.nvm/versions/node/v17.9.1/bin/node /etc/metlo-ingestor/ingestors/suricata/dist/index.js -s /etc/suricata-logs/eve.sock -e /opt/metlo/credentials
+
+[Install]
+WantedBy=multi-user.target
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		alert http !169.254.169.254 any -> any any (msg:"TEST"; flow:established,to_client; http.response_body; pcre:/./; sid:1; rev:1; threshold: type limit, track by_rule, seconds 1, count 30;)