v23.03.0 development

Dockerfiles/arkime.Dockerfile

@@ -4,7 +4,7 @@ FROM debian:11-slim AS build
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ENV ARKIME_VERSION "v4.1.0"
+ENV ARKIME_VERSION "v4.2.0"
 ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no

Dockerfiles/dashboards-helper.Dockerfile

@@ -47,10 +47,10 @@ ENV DASHBOARDS_URL $DASHBOARDS_URL
 ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE
 ENV PATH="/data:${PATH}"
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 ENV ECS_RELEASES_URL "https://api.github.com/repos/elastic/ecs/releases/latest"

Dockerfiles/dashboards.Dockerfile

@@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"
 
 ENV TERM xterm
 
-ARG OPENSEARCH_VERSION="2.5.0"
+ARG OPENSEARCH_VERSION="2.6.0"
 ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION
 
-ARG OPENSEARCH_DASHBOARDS_VERSION="2.5.0"
+ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0"
 ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION
 
 # base system dependencies for checking out and building plugins
@@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \
 
 # runtime ##################################################################
 
-FROM opensearchproject/opensearch-dashboards:2.5.0
+FROM opensearchproject/opensearch-dashboards:2.6.0
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -122,7 +122,7 @@ RUN yum upgrade -y && \
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
     cd /usr/share/opensearch-dashboards/plugins && \
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \
-    /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \
+    # TODO: when 2.6.0 is released /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \
     # trying to see if things still work if these are owned by root (to avoid a costly chown on container startup)
     chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \
                                 /usr/share/opensearch-dashboards/node_modules/* \

Dockerfiles/file-monitor.Dockerfile

@@ -79,7 +79,7 @@ ENV YARA_VERSION "4.2.3"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_SRC_DIR "/yara-rules-src"
 ENV YARA_RULES_DIR "/yara-rules"
-ENV CAPA_VERSION "4.0.1"
+ENV CAPA_VERSION "5.0.0"
 ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
 ENV CAPA_DIR "/opt/capa"
 ENV CAPA_BIN "${CAPA_DIR}/capa"
@@ -89,10 +89,10 @@ ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
 ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
 ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/

Dockerfiles/filebeat.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.6.1
+FROM docker.elastic.co/beats/filebeat-oss:8.6.2
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -57,10 +57,10 @@ ARG FILEBEAT_TCP_PARSE_TARGET_FIELD=""
 ARG FILEBEAT_TCP_PARSE_DROP_FIELD=""
 ARG FILEBEAT_TCP_TAG="_malcolm_beats"
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 ENV TINI_VERSION v0.19.0

Dockerfiles/logstash.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0
+FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.6.1
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -46,7 +46,8 @@ USER root
 
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
 
-RUN apt-get -q update && \
+RUN set -x && \
+    apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get -y --no-install-recommends install \
         gettext \
@@ -57,6 +58,8 @@ RUN apt-get -q update && \
         tini && \
     chmod +x /usr/bin/tini && \
     pip3 install ipaddress supervisor manuf pyyaml && \
+    export JAVA_HOME=/usr/share/logstash/jdk && \
+    /usr/share/logstash/vendor/jruby/bin/jruby -S gem install bundler && \
     echo "gem 'lru_cache'" >> /usr/share/logstash/Gemfile && \
     /usr/share/logstash/bin/ruby -S bundle install && \
     logstash-plugin install --preserve logstash-filter-translate logstash-filter-cidr logstash-filter-dns \

Dockerfiles/netbox.Dockerfile

@@ -22,10 +22,10 @@ ENV PUSER "boxer"
 ENV PGROUP "boxer"
 ENV PUSER_PRIV_DROP true
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 ENV NETBOX_DEVICETYPE_LIBRARY_URL "https://codeload.github.com/netbox-community/devicetype-library/tar.gz/master"
@@ -62,6 +62,8 @@ RUN apt-get -q update && \
       usermod -a -G tty ${PUSER} && \
     mkdir -p /opt/unit "${NETBOX_DEVICETYPE_LIBRARY_PATH}" && \
     chown -R $PUSER:$PGROUP /etc/netbox /opt/unit /opt/netbox && \
+    # trying to see if things still work if these are owned by root (to avoid a costly chown on container startup)
+    chown --silent -R root:root /opt/netbox/venv/* && \
     cd "$(dirname "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" && \
         curl -sSL "$NETBOX_DEVICETYPE_LIBRARY_URL" | tar xzvf - -C ./"$(basename "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" --strip-components 1 && \
     mkdir -p /opt/netbox/netbox/$BASE_PATH && \

Dockerfiles/opensearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.5.0
+FROM opensearchproject/opensearch:2.6.0
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/pcap-monitor.Dockerfile

@@ -59,7 +59,7 @@ RUN apt-get -q update && \
       vim-tiny && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    pip3 install --no-cache-dir opensearch-py opensearch-dsl pyzmq pyinotify python-magic requests && \
+    pip3 install --no-cache-dir opensearch-py pyzmq pyinotify python-magic requests && \
     groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
       useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}
 

Dockerfiles/suricata.Dockerfile

@@ -27,10 +27,10 @@ ENV PGROUP "suricata"
 # a final check in docker_entrypoint.sh before startup
 ENV PUSER_PRIV_DROP false
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 ENV YQ_VERSION "4.24.2"

Dockerfiles/zeek.Dockerfile

@@ -31,15 +31,15 @@ ENV PUSER_PRIV_DROP false
 
 # for download and install
 ARG ZEEK_LTS=true
-ARG ZEEK_VERSION=5.0.6-0
+ARG ZEEK_VERSION=5.0.7-0
 
 ENV ZEEK_LTS $ZEEK_LTS
 ENV ZEEK_VERSION $ZEEK_VERSION
 
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
 ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 # for build
@@ -165,8 +165,8 @@ ADD shared/bin/nic-capture-setup.sh /usr/local/bin/
 # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
 ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22
 ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
-ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 23
-ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
+ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25
+ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
 
 RUN mkdir -p /tmp/logs && \
     cd /tmp/logs && \

_config.yml

@@ -4,7 +4,7 @@ description: A powerful, easily deployable network traffic analysis tool suite
 logo: docs/images/logo/Malcolm_outline_banner_dark.png
 remote_theme: pages-themes/[email protected]
 external_download_url: https://malcolm.fyi/docs/download.html
-youtube_url: https://www.youtube.com/c/MalcolmNetworkTrafficAnalysisToolSuite
+youtube_url: https://www.youtube.com/@MalcolmNetworkTrafficAnalysis
 mastodon:
   id:
   url:

api/project/__init__.py

@@ -1,7 +1,6 @@
 import dateparser
 import json
 import malcolm_common
-import opensearch_dsl
 import opensearchpy
 import os
 import pytz
@@ -167,15 +166,15 @@
     else defaultdict(lambda: None)
 )
 if opensearchCreds['user'] is not None:
-    opensearchDslHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
+    opensearchHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
     opensearchReqHttpAuth = HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
 else:
-    opensearchDslHttpAuth = None
+    opensearchHttpAuth = None
     opensearchReqHttpAuth = None
 
-opensearch_dsl.connections.create_connection(
+opensearchClient = opensearchpy.OpenSearch(
     hosts=[opensearchUrl],
-    http_auth=opensearchDslHttpAuth,
+    http_auth=opensearchHttpAuth,
     verify_certs=opensearchSslVerify,
     ssl_assert_hostname=False,
     ssl_show_warn=False,
@@ -333,7 +332,7 @@ def filtertime(search, args, default_from="1 day ago", default_to="now"):
 
     Parameters
     ----------
-    search : opensearch_dsl.Search
+    search : opensearchpy.Search
         The object representing the OpenSearch Search query
     args : dict
         The dictionary which should contain 'from' and 'to' times (see gettimes)
@@ -377,7 +376,7 @@ def filtervalues(search, args):
 
     Parameters
     ----------
-    search : opensearch_dsl.Search
+    search : opensearchpy.Search
         The object representing the OpenSearch Search query
     args : dict
         The dictionary which should contain 'filter' (see getfilters)
@@ -413,7 +412,7 @@ def filtervalues(search, args):
                     )
                 else:
                     # field does not exist ("is null")
-                    s = s.filter('bool', must_not=opensearch_dsl.Q('exists', field=fieldname))
+                    s = s.filter('bool', must_not=opensearchpy.helpers.query.Q('exists', field=fieldname))
 
     if debugApi:
         print(f'filtervalues: {json.dumps(s.to_dict())}')
@@ -442,8 +441,11 @@ def bucketfield(fieldname, current_request, urls=None):
     fields
         the name of the field(s) on which the aggregation was performed
     """
-    s = opensearch_dsl.Search(
-        using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_INDEX_PATTERN"]
+    global opensearchClient
+
+    s = opensearchpy.Search(
+        using=opensearchClient,
+        index=app.config["ARKIME_INDEX_PATTERN"],
     ).extra(size=0)
     args = get_request_arguments(current_request)
     start_time_ms, end_time_ms, s = filtertime(s, args)
@@ -523,10 +525,13 @@ def document(index):
     results
         array of the documents retrieved (up to 'limit')
     """
+    global opensearchClient
+
     args = get_request_arguments(request)
-    s = opensearch_dsl.Search(using=opensearch_dsl.connections.get_connection(), index=index).extra(
-        size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"]))
-    )
+    s = opensearchpy.Search(
+        using=opensearchClient,
+        index=index,
+    ).extra(size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"])))
     start_time_ms, end_time_ms, s = filtertime(s, args, default_from="1970-1-1", default_to="now")
     filters, s = filtervalues(s, args)
     return jsonify(
@@ -574,6 +579,8 @@ def fields():
     fields
         A dict of dicts where key is the field name and value may contain 'description' and 'type'
     """
+    global opensearchClient
+
     args = get_request_arguments(request)
 
     templateName = args['template'] if 'template' in args else app.config["MALCOLM_TEMPLATE"]
@@ -585,8 +592,9 @@ def fields():
     if arkimeFields:
         try:
             # get fields from Arkime's field's table
-            s = opensearch_dsl.Search(
-                using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_FIELDS_INDEX"]
+            s = opensearchpy.Search(
+                using=opensearchClient,
+                index=app.config["ARKIME_FIELDS_INDEX"],
             ).extra(size=5000)
             for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]:
                 if (fieldname := deep_get(hit, ['dbField2'])) and (fieldname not in fields):
@@ -697,6 +705,8 @@ def version():
     opensearch_health
         a JSON structure containing OpenSearch cluster health
     """
+    global opensearchClient
+
     return jsonify(
         version=app.config["MALCOLM_VERSION"],
         built=app.config["BUILD_DATE"],
@@ -706,7 +716,7 @@ def version():
             auth=opensearchReqHttpAuth,
             verify=opensearchSslVerify,
         ).json(),
-        opensearch_health=opensearch_dsl.connections.get_connection().cluster.health(),
+        opensearch_health=opensearchClient.cluster.health(),
     )
 
 
@@ -783,6 +793,8 @@ def event():
     status
         the JSON-formatted OpenSearch response from indexing/updating the alert record
     """
+    global opensearchClient
+
     alert = {}
     idxResponse = {}
     data = get_request_arguments(request)
@@ -880,7 +892,7 @@ def event():
                     alert['event']['hits'] = hitCount
 
         docDateStr = dateparser.parse(alert['@timestamp']).strftime('%y%m%d')
-        idxResponse = opensearch_dsl.connections.get_connection().index(
+        idxResponse = opensearchClient.index(
             index=f"{app.config['ARKIME_INDEX_PATTERN'].rstrip('*')}{docDateStr}",
             id=f"{docDateStr}-{alert['event']['id']}",
             body=alert,

api/requirements.txt

@@ -1,8 +1,7 @@
 pytz==2021.3
 Flask==2.0.2
 gunicorn==20.1.0
-opensearch-py==2.1.1
-opensearch-dsl==2.0.1
+opensearch-py==2.2.0
 requests==2.26.0
 regex==2022.3.2
 dateparser==1.1.1