Merge branch 'development' of https://github.com/mmguero-dev/Malcolm …

…into v643_merge_idaholab
mmguero-dev · Dec 5, 2022 · 476d941 · 476d941
2 parents 62518b1 + 8f25215
commit 476d941
Show file tree

Hide file tree

Showing 33 changed files with 1,208 additions and 205 deletions.
diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.16
+FROM alpine:3.17
 
 # Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.5.1
+FROM docker.elastic.co/beats/filebeat-oss:8.5.2
 
 # Copyright (c) 2022 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

diff --git a/Dockerfiles/name-map-ui.Dockerfile b/Dockerfiles/name-map-ui.Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.16
+FROM alpine:3.17
 
 # Copyright (c) 2022 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -26,13 +26,13 @@ ENV LISTJS_VERSION v1.5.0
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk --no-cache add bash php8 php8-fpm php8-mysqli php8-json php8-openssl php8-curl php8-fileinfo \
-    php8-zlib php8-xml php8-phar php8-intl php8-dom php8-xmlreader php8-ctype php8-session \
-    php8-mbstring php8-gd nginx supervisor curl inotify-tools file psmisc shadow openssl tini
+    apk --no-cache add bash php81 php81-fpm php81-mysqli php81-json php81-openssl php81-curl php81-fileinfo \
+    php81-zlib php81-xml php81-phar php81-intl php81-dom php81-xmlreader php81-ctype php81-session \
+    php81-mbstring php81-gd nginx supervisor curl inotify-tools file psmisc shadow openssl tini
 
 COPY name-map-ui/config/nginx.conf /etc/nginx/nginx.conf
-COPY name-map-ui/config/fpm-pool.conf /etc/php8/php-fpm.d/www.conf
-COPY name-map-ui/config/php.ini /etc/php8/conf.d/custom.ini
+COPY name-map-ui/config/fpm-pool.conf /etc/php81/php-fpm.d/www.conf
+COPY name-map-ui/config/php.ini /etc/php81/conf.d/custom.ini
 COPY name-map-ui/config/supervisord.conf /etc/supervisord.conf
 COPY name-map-ui/config/supervisor_logstash_ctl.conf /etc/supervisor/logstash/supervisord.conf
 COPY name-map-ui/config/supervisor_netbox_ctl.conf /etc/supervisor/netbox/supervisord.conf

diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile
@@ -28,6 +28,8 @@ ENV SUPERCRONIC "supercronic-linux-amd64"
 ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
+ENV NETBOX_DEVICETYPE_LIBRARY_URL "https://codeload.github.com/netbox-community/devicetype-library/tar.gz/master"
+
 ARG NETBOX_DEFAULT_SITE=Malcolm
 ARG NETBOX_CRON=false
 
@@ -54,10 +56,12 @@ RUN apt-get -q update && \
       apt-get clean && \
       rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     groupadd --gid ${DEFAULT_GID} ${PUSER} && \
-    useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
-    usermod -a -G tty ${PUSER} && \
-    mkdir -p /opt/unit && \
+      useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
+      usermod -a -G tty ${PUSER} && \
+    mkdir -p /opt/unit /opt/netbox-devicetype-library && \
     chown -R $PUSER:$PGROUP /etc/netbox /opt/unit /opt/netbox && \
+    cd /opt && \
+        curl -sSL "$NETBOX_DEVICETYPE_LIBRARY_URL" | tar xzvf - -C ./netbox-devicetype-library --strip-components 1 && \
     mkdir -p /opt/netbox/netbox/$BASE_PATH && \
       mv /opt/netbox/netbox/static /opt/netbox/netbox/$BASE_PATH/static && \
       jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq ".routes[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \

diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile
@@ -25,13 +25,15 @@ WORKDIR /site
 # build documentation, remove unnecessary files, then massage a bit to work nicely with NGINX (which will be serving it)
 RUN find /site -type f -name "*.md" -exec sed -i "s/{{[[:space:]]*site.github.build_revision[[:space:]]*}}/$VCS_REVISION/g" "{}" \; && \
     ( [ -n "${GITHUB_TOKEN}" ] && export JEKYLL_GITHUB_TOKEN="${GITHUB_TOKEN}" || true ) && \
+    sed -i "s/^\(show_downloads:\).*/\1 false/" /site/_config.yml && \
+    sed -i -e "/^mastodon:/,+2d" /site/_config.yml && \
     docker-entrypoint.sh bundle exec jekyll build && \
     find /site/_site -type f -name "*.md" -delete && \
     find /site/_site -type f -name "*.html" -exec sed -i "s@/\(docs\|assets\)@/readme/\1@g" "{}" \; && \
     find /site/_site -type f -name "*.html" -exec sed -i 's@\(href=\)"/"@\1"/readme/"@g' "{}" \;
 
 # build NGINX image
-FROM alpine:3.16
+FROM alpine:3.17
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -141,7 +143,7 @@ RUN set -x ; \
   " ; \
   apk update --no-cache; \
   apk upgrade --no-cache; \
-  apk add --no-cache curl shadow; \
+  apk add --no-cache curl shadow libressl; \
   addgroup -g ${DEFAULT_GID} -S ${PGROUP} ; \
   adduser -S -D -H -u ${DEFAULT_UID} -h /var/cache/nginx -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
   addgroup ${PUSER} shadow ; \

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
@@ -31,7 +31,7 @@ ENV PUSER_PRIV_DROP false
 
 # for download and install
 ARG ZEEK_LTS=true
-ARG ZEEK_VERSION=5.0.3-0
+ARG ZEEK_VERSION=5.0.4-0
 
 ENV ZEEK_LTS $ZEEK_LTS
 ENV ZEEK_VERSION $ZEEK_VERSION

diff --git a/api/requirements.txt b/api/requirements.txt
@@ -1,7 +1,7 @@
 pytz==2021.3
 Flask==2.0.2
 gunicorn==20.1.0
-opensearch-py==2.0.0
+opensearch-py==2.0.1
 opensearch-dsl==2.0.1
 requests==2.26.0
 regex==2022.3.2

diff --git a/dashboards/templates/composable/component/arkime.json b/dashboards/templates/composable/component/arkime.json
@@ -16,6 +16,7 @@
         "protocol": { "type": "keyword" },
         "quic.host": { "type": "keyword" },
         "quic.version": { "type": "keyword" },
+        "rootId": { "type": "keyword" },
         "source.geo.country_code2": { "type": "keyword" },
         "source.geo.country_code3": { "type": "keyword" },
         "source.geo.dma_code": { "type": "short" },

diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
@@ -352,7 +352,7 @@ x-pcap-capture-variables: &pcap-capture-variables
 
 services:
   opensearch:
-    image: malcolmnetsec/opensearch:6.4.2
+    image: malcolmnetsec/opensearch:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -383,17 +383,17 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./.opensearch.primary.curlrc:/var/local/opensearch.primary.curlrc:ro
       - ./.opensearch.secondary.curlrc:/var/local/opensearch.secondary.curlrc:ro
-      - ./opensearch/opensearch.keystore:/usr/share/opensearch/config/opensearch.keystore:rw
       - ./opensearch:/usr/share/opensearch/data:delegated
       - ./opensearch-backup:/opt/opensearch/backup:delegated
+      - ./opensearch/opensearch.keystore:/usr/share/opensearch/config/opensearch.keystore:rw
     healthcheck:
       test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9200"]
       interval: 30s
       timeout: 15s
       retries: 3
       start_period: 180s
   dashboards-helper:
-    image: malcolmnetsec/dashboards-helper:6.4.2
+    image: malcolmnetsec/dashboards-helper:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -424,7 +424,7 @@ services:
       retries: 3
       start_period: 30s
   dashboards:
-    image: malcolmnetsec/dashboards:6.4.2
+    image: malcolmnetsec/dashboards:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -449,7 +449,7 @@ services:
       retries: 3
       start_period: 210s
   logstash:
-    image: malcolmnetsec/logstash-oss:6.4.2
+    image: malcolmnetsec/logstash-oss:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -492,7 +492,7 @@ services:
       retries: 3
       start_period: 600s
   filebeat:
-    image: malcolmnetsec/filebeat-oss:6.4.2
+    image: malcolmnetsec/filebeat-oss:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -531,7 +531,7 @@ services:
       retries: 3
       start_period: 60s
   arkime:
-    image: malcolmnetsec/arkime:6.4.2
+    image: malcolmnetsec/arkime:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -569,7 +569,7 @@ services:
       retries: 3
       start_period: 210s
   zeek:
-    image: malcolmnetsec/zeek:6.4.2
+    image: malcolmnetsec/zeek:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -608,7 +608,7 @@ services:
       retries: 3
       start_period: 60s
   zeek-live:
-    image: malcolmnetsec/zeek:6.4.2
+    image: malcolmnetsec/zeek:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -640,7 +640,7 @@ services:
       - ./zeek-logs/extract_files:/zeek/extract_files
       - ./zeek/intel:/opt/zeek/share/zeek/site/intel
   suricata:
-    image: malcolmnetsec/suricata:6.4.2
+    image: malcolmnetsec/suricata:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -677,7 +677,7 @@ services:
       retries: 3
       start_period: 120s
   suricata-live:
-    image: malcolmnetsec/suricata:6.4.2
+    image: malcolmnetsec/suricata:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -704,7 +704,7 @@ services:
       - ./suricata-logs:/var/log/suricata
       - ./suricata/rules:/opt/suricata/rules:ro
   file-monitor:
-    image: malcolmnetsec/file-monitor:6.4.2
+    image: malcolmnetsec/file-monitor:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -728,7 +728,7 @@ services:
       retries: 3
       start_period: 60s
   pcap-capture:
-    image: malcolmnetsec/pcap-capture:6.4.2
+    image: malcolmnetsec/pcap-capture:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -750,7 +750,7 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./pcap/upload:/pcap
   pcap-monitor:
-    image: malcolmnetsec/pcap-monitor:6.4.2
+    image: malcolmnetsec/pcap-monitor:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -776,7 +776,7 @@ services:
       retries: 3
       start_period: 90s
   upload:
-    image: malcolmnetsec/file-upload:6.4.2
+    image: malcolmnetsec/file-upload:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -804,7 +804,7 @@ services:
       retries: 3
       start_period: 60s
   htadmin:
-    image: malcolmnetsec/htadmin:6.4.2
+    image: malcolmnetsec/htadmin:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -828,7 +828,7 @@ services:
       retries: 3
       start_period: 60s
   freq:
-    image: malcolmnetsec/freq:6.4.2
+    image: malcolmnetsec/freq:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -849,7 +849,7 @@ services:
       retries: 3
       start_period: 60s
   name-map-ui:
-    image: malcolmnetsec/name-map-ui:6.4.2
+    image: malcolmnetsec/name-map-ui:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -872,7 +872,7 @@ services:
       retries: 3
       start_period: 60s
   netbox:
-    image: malcolmnetsec/netbox:6.4.2
+    image: malcolmnetsec/netbox:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -903,7 +903,7 @@ services:
       retries: 3
       start_period: 120s
   netbox-postgres:
-    image: malcolmnetsec/postgresql:6.4.2
+    image: malcolmnetsec/postgresql:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -920,13 +920,13 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./netbox/postgres:/var/lib/postgresql/data:rw
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ]
+      test: [ "CMD-SHELL", "[[ $${NETBOX_POSTGRES_DISABLED} == 'true' ]] || pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ]
       interval: 60s
       timeout: 15s
       retries: 3
       start_period: 45s
   netbox-redis:
-    image: malcolmnetsec/redis:6.4.2
+    image: malcolmnetsec/redis:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -947,13 +947,13 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./netbox/redis:/data
     healthcheck:
-      test: ["CMD-SHELL", "pidof redis-server || exit 1" ]
+      test: ["CMD-SHELL", "[[ $${NETBOX_REDIS_DISABLED} == 'true' ]] || ( pidof redis-server || exit 1 )" ]
       interval: 60s
       timeout: 15s
       retries: 3
       start_period: 45s
   netbox-redis-cache:
-    image: malcolmnetsec/redis:6.4.2
+    image: malcolmnetsec/redis:6.4.3
     restart: "no"
     stdin_open: false
     tty: true
@@ -973,13 +973,13 @@ services:
     volumes:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
     healthcheck:
-      test: ["CMD-SHELL", "pidof redis-server || exit 1" ]
+      test: ["CMD-SHELL", "[[ $${NETBOX_REDIS_DISABLED} == 'true' ]] || ( pidof redis-server || exit 1 )" ]
       interval: 60s
       timeout: 15s
       retries: 3
       start_period: 45s
   api:
-    image: malcolmnetsec/api:6.4.2
+    image: malcolmnetsec/api:6.4.3
     command: gunicorn --bind 0:5000 manage:app
     restart: "no"
     stdin_open: false
@@ -1002,7 +1002,7 @@ services:
       retries: 3
       start_period: 60s
   nginx-proxy:
-    image: malcolmnetsec/nginx-proxy:6.4.2
+    image: malcolmnetsec/nginx-proxy:6.4.3
     restart: "no"
     stdin_open: false
     tty: true