From bffcbcabbfe0f6cdf0a32abd683606f4ca013457 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 3 Feb 2023 15:22:05 -0700 Subject: [PATCH 01/30] bump development for v23.02.1 --- docker-compose-standalone.yml | 46 +++++++++++++++++----------------- docker-compose.yml | 46 +++++++++++++++++----------------- docs/download.md | 4 +-- docs/hedgehog-iso-build.md | 2 +- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 40 ++++++++++++++--------------- docs/ubuntu-install-example.md | 40 ++++++++++++++--------------- 7 files changed, 90 insertions(+), 90 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 7d52feba7..0826c93ec 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -359,7 +359,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: opensearch: - image: malcolmnetsec/opensearch:23.02.0 + image: malcolmnetsec/opensearch:23.02.1 restart: "no" stdin_open: false tty: true @@ -400,7 +400,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: malcolmnetsec/dashboards-helper:23.02.0 + image: malcolmnetsec/dashboards-helper:23.02.1 restart: "no" stdin_open: false tty: true @@ -431,7 +431,7 @@ services: retries: 3 start_period: 30s dashboards: - image: malcolmnetsec/dashboards:23.02.0 + image: malcolmnetsec/dashboards:23.02.1 restart: "no" stdin_open: false tty: true @@ -456,7 +456,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:23.02.0 + image: malcolmnetsec/logstash-oss:23.02.1 restart: "no" stdin_open: false tty: true @@ -499,7 +499,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:23.02.0 + image: malcolmnetsec/filebeat-oss:23.02.1 restart: "no" stdin_open: false tty: true @@ -538,7 +538,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:23.02.0 + image: malcolmnetsec/arkime:23.02.1 restart: "no" stdin_open: false tty: true @@ -576,7 +576,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:23.02.0 + image: malcolmnetsec/zeek:23.02.1 restart: "no" stdin_open: false tty: true @@ -615,7 +615,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: malcolmnetsec/zeek:23.02.0 + image: malcolmnetsec/zeek:23.02.1 restart: "no" stdin_open: false tty: true @@ -647,7 +647,7 @@ services: - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel suricata: - image: malcolmnetsec/suricata:23.02.0 + image: malcolmnetsec/suricata:23.02.1 restart: "no" stdin_open: false tty: true @@ -684,7 +684,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: malcolmnetsec/suricata:23.02.0 + image: malcolmnetsec/suricata:23.02.1 restart: "no" stdin_open: false tty: true @@ -711,7 +711,7 @@ services: - ./suricata-logs:/var/log/suricata - ./suricata/rules:/opt/suricata/rules:ro file-monitor: - image: malcolmnetsec/file-monitor:23.02.0 + image: malcolmnetsec/file-monitor:23.02.1 restart: "no" stdin_open: false tty: true @@ -735,7 +735,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:23.02.0 + image: malcolmnetsec/pcap-capture:23.02.1 restart: "no" stdin_open: false tty: true @@ -757,7 +757,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: malcolmnetsec/pcap-monitor:23.02.0 + image: malcolmnetsec/pcap-monitor:23.02.1 restart: "no" stdin_open: false tty: true @@ -783,7 +783,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:23.02.0 + image: malcolmnetsec/file-upload:23.02.1 restart: "no" stdin_open: false tty: true @@ -811,7 +811,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:23.02.0 + image: malcolmnetsec/htadmin:23.02.1 restart: "no" stdin_open: false tty: true @@ -835,7 +835,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:23.02.0 + image: malcolmnetsec/freq:23.02.1 restart: "no" stdin_open: false tty: true @@ -856,7 +856,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:23.02.0 + image: malcolmnetsec/name-map-ui:23.02.1 restart: "no" stdin_open: false tty: true @@ -877,7 +877,7 @@ services: retries: 3 start_period: 60s netbox: - image: malcolmnetsec/netbox:23.02.0 + image: malcolmnetsec/netbox:23.02.1 restart: "no" stdin_open: false tty: true @@ -908,7 +908,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: malcolmnetsec/postgresql:23.02.0 + image: malcolmnetsec/postgresql:23.02.1 restart: "no" stdin_open: false tty: true @@ -931,7 +931,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: malcolmnetsec/redis:23.02.0 + image: malcolmnetsec/redis:23.02.1 restart: "no" stdin_open: false tty: true @@ -958,7 +958,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: malcolmnetsec/redis:23.02.0 + image: malcolmnetsec/redis:23.02.1 restart: "no" stdin_open: false tty: true @@ -984,7 +984,7 @@ services: retries: 3 start_period: 45s api: - image: malcolmnetsec/api:23.02.0 + image: malcolmnetsec/api:23.02.1 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -1007,7 +1007,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:23.02.0 + image: malcolmnetsec/nginx-proxy:23.02.1 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index 9b4b2753d..f305df50d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: malcolmnetsec/opensearch:23.02.0 + image: malcolmnetsec/opensearch:23.02.1 restart: "no" stdin_open: false tty: true @@ -406,7 +406,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: malcolmnetsec/dashboards-helper:23.02.0 + image: malcolmnetsec/dashboards-helper:23.02.1 restart: "no" stdin_open: false tty: true @@ -440,7 +440,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: malcolmnetsec/dashboards:23.02.0 + image: malcolmnetsec/dashboards:23.02.1 restart: "no" stdin_open: false tty: true @@ -468,7 +468,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:23.02.0 + image: malcolmnetsec/logstash-oss:23.02.1 restart: "no" stdin_open: false tty: true @@ -518,7 +518,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:23.02.0 + image: malcolmnetsec/filebeat-oss:23.02.1 restart: "no" stdin_open: false tty: true @@ -560,7 +560,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:23.02.0 + image: malcolmnetsec/arkime:23.02.1 restart: "no" stdin_open: false tty: true @@ -604,7 +604,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:23.02.0 + image: malcolmnetsec/zeek:23.02.1 restart: "no" stdin_open: false tty: true @@ -647,7 +647,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:23.02.0 + image: malcolmnetsec/zeek:23.02.1 restart: "no" stdin_open: false tty: true @@ -683,7 +683,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:23.02.0 + image: malcolmnetsec/suricata:23.02.1 restart: "no" stdin_open: false tty: true @@ -723,7 +723,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:23.02.0 + image: malcolmnetsec/suricata:23.02.1 restart: "no" stdin_open: false tty: true @@ -753,7 +753,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:23.02.0 + image: malcolmnetsec/file-monitor:23.02.1 restart: "no" stdin_open: false tty: true @@ -780,7 +780,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:23.02.0 + image: malcolmnetsec/pcap-capture:23.02.1 restart: "no" stdin_open: false tty: true @@ -805,7 +805,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:23.02.0 + image: malcolmnetsec/pcap-monitor:23.02.1 restart: "no" stdin_open: false tty: true @@ -834,7 +834,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:23.02.0 + image: malcolmnetsec/file-upload:23.02.1 restart: "no" stdin_open: false tty: true @@ -862,7 +862,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:23.02.0 + image: malcolmnetsec/htadmin:23.02.1 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -889,7 +889,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:23.02.0 + image: malcolmnetsec/freq:23.02.1 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -913,7 +913,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:23.02.0 + image: malcolmnetsec/name-map-ui:23.02.1 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -937,7 +937,7 @@ services: retries: 3 start_period: 60s netbox: - image: malcolmnetsec/netbox:23.02.0 + image: malcolmnetsec/netbox:23.02.1 build: context: . dockerfile: Dockerfiles/netbox.Dockerfile @@ -972,7 +972,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: malcolmnetsec/postgresql:23.02.0 + image: malcolmnetsec/postgresql:23.02.1 build: context: . dockerfile: Dockerfiles/postgresql.Dockerfile @@ -998,7 +998,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: malcolmnetsec/redis:23.02.0 + image: malcolmnetsec/redis:23.02.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -1028,7 +1028,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: malcolmnetsec/redis:23.02.0 + image: malcolmnetsec/redis:23.02.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -1057,7 +1057,7 @@ services: retries: 3 start_period: 45s api: - image: malcolmnetsec/api:23.02.0 + image: malcolmnetsec/api:23.02.1 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -1086,7 +1086,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:23.02.0 + image: malcolmnetsec/nginx-proxy:23.02.1 restart: "no" stdin_open: false tty: true diff --git a/docs/download.md b/docs/download.md index 79d274a95..b0b2acb21 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.02.0.iso](/iso/malcolm-23.02.0.iso) (5.3GiB) | [`2d43ad3023e19460852ba5bf28e4b6d35072308bef07945baa4284bbf77fbb09`](/iso/malcolm-23.02.0.iso.sha256.txt) | +| [malcolm-23.02.1.iso](/iso/malcolm-23.02.1.iso) (5.3GiB) | [`xxxxxxxx`](/iso/malcolm-23.02.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.02.0.iso](/iso/hedgehog-23.02.0.iso) (2.3GiB) | [`21f0a0ae746c993853c26bbe88995f25fc1784192155616c14f2b4645c14ef44`](/iso/hedgehog-23.02.0.iso.sha256.txt) | +| [hedgehog-23.02.1.iso](/iso/hedgehog-23.02.1.iso) (2.3GiB) | [`xxxxxxxx`](/iso/hedgehog-23.02.1.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 29b0e1dbd..8f0167a7f 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.02.0.iso" +Finished, created "/sensor-build/hedgehog-23.02.1.iso" … ``` diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index e5009970d..7df41506d 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.02.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.02.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 96278aa73..5cedaaa8b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -53,26 +53,26 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 23.02.0 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 23.02.0 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 23.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 23.02.0 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/file-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 23.02.0 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/filebeat-oss 23.02.0 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/freq 23.02.0 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 23.02.0 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 23.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 23.02.0 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/netbox 23.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -malcolmnetsec/nginx-proxy 23.02.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 23.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 23.02.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/postgresql 23.02.0 xxxxxxxxxxxx 3 days ago 268MB -malcolmnetsec/redis 23.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -malcolmnetsec/suricata 23.02.0 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 23.02.0 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 23.02.1 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 23.02.1 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 23.02.1 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 23.02.1 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/file-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 23.02.1 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/filebeat-oss 23.02.1 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/freq 23.02.1 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 23.02.1 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 23.02.1 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 23.02.1 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/netbox 23.02.1 xxxxxxxxxxxx 3 days ago 1.01GB +malcolmnetsec/nginx-proxy 23.02.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 23.02.1 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 23.02.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/postgresql 23.02.1 xxxxxxxxxxxx 3 days ago 268MB +malcolmnetsec/redis 23.02.1 xxxxxxxxxxxx 3 days ago 34.2MB +malcolmnetsec/suricata 23.02.1 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 23.02.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index d09d25bdb..5850b6d33 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -261,26 +261,26 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 23.02.0 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 23.02.0 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 23.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 23.02.0 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/file-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 23.02.0 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/filebeat-oss 23.02.0 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/freq 23.02.0 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 23.02.0 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 23.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 23.02.0 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/netbox 23.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -malcolmnetsec/nginx-proxy 23.02.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 23.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 23.02.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/postgresql 23.02.0 xxxxxxxxxxxx 3 days ago 268MB -malcolmnetsec/redis 23.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -malcolmnetsec/suricata 23.02.0 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 23.02.0 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 23.02.1 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 23.02.1 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 23.02.1 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 23.02.1 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/file-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 23.02.1 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/filebeat-oss 23.02.1 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/freq 23.02.1 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 23.02.1 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 23.02.1 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 23.02.1 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/netbox 23.02.1 xxxxxxxxxxxx 3 days ago 1.01GB +malcolmnetsec/nginx-proxy 23.02.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 23.02.1 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 23.02.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/postgresql 23.02.1 xxxxxxxxxxxx 3 days ago 268MB +malcolmnetsec/redis 23.02.1 xxxxxxxxxxxx 3 days ago 34.2MB +malcolmnetsec/suricata 23.02.1 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 23.02.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. From 602d7b543114872218ca00f90a96c46a6e6e1009 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 3 Feb 2023 15:29:22 -0700 Subject: [PATCH 02/30] replace misc/scan.zeek with ncsa/bro-simple-scan (https://github.com/zeek/zeek/blob/cdadc329859810244244c8800f0102543e4f134f/NEWS#L540-L541) --- Dockerfiles/zeek.Dockerfile | 4 ++-- .../config/includes.chroot/usr/local/etc/zeek/local.zeek | 1 - shared/bin/zeek_install_plugins.sh | 1 + zeek/config/local.zeek | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index ce35d248f..2fc83705e 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -165,8 +165,8 @@ ADD shared/bin/nic-capture-setup.sh /usr/local/bin/ # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22 ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" -ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 23 -ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" +ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 24 +ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ cd /tmp/logs && \ diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 29ef4f319..917c55401 100644 --- a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -25,7 +25,6 @@ redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; @load tuning/defaults -@load misc/scan @load frameworks/software/vulnerable @load frameworks/software/version-changes @load frameworks/software/windows-version-detection diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index bf7523163..b91b62ebe 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -108,6 +108,7 @@ ZKG_GITHUB_URLS=( "https://github.com/corelight/zerologon" "https://github.com/cybera/zeek-sniffpass" "https://github.com/mmguero-dev/bzar" + "https://github.com/ncsa/bro-simple-scan" "https://github.com/precurse/zeek-httpattacks" "https://github.com/salesforce/GQUIC_Protocol_Analyzer" "https://github.com/salesforce/hassh" diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index 709cbcf2a..42df7ff2e 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -25,7 +25,6 @@ redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; @load tuning/defaults -@load misc/scan @load frameworks/software/vulnerable @load frameworks/software/version-changes @load frameworks/software/windows-version-detection From adaebdcc09fae87927f7489f5c134e6bc03c5d2e Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 3 Feb 2023 15:50:29 -0700 Subject: [PATCH 03/30] Added dependency for simple scan script --- Dockerfiles/zeek.Dockerfile | 4 ++-- shared/bin/zeek_install_plugins.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 2fc83705e..3df19c68b 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -165,8 +165,8 @@ ADD shared/bin/nic-capture-setup.sh /usr/local/bin/ # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22 ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" -ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 24 -ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" +ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25 +ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ cd /tmp/logs && \ diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index b91b62ebe..f3840098c 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -108,6 +108,7 @@ ZKG_GITHUB_URLS=( "https://github.com/corelight/zerologon" "https://github.com/cybera/zeek-sniffpass" "https://github.com/mmguero-dev/bzar" + "https://github.com/ncsa/bro-is-darknet" "https://github.com/ncsa/bro-simple-scan" "https://github.com/precurse/zeek-httpattacks" "https://github.com/salesforce/GQUIC_Protocol_Analyzer" From 0dda49bef8f9e6fa639b76df608f868d915a2f40 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 13 Feb 2023 09:08:36 -0700 Subject: [PATCH 04/30] bump capa to v5.0.0 and fluent-bit to 2.0.9 for windows convenience script --- Dockerfiles/file-monitor.Dockerfile | 2 +- scripts/third-party-logs/fluent-bit-setup.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index ea1055ff0..b13613f72 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -79,7 +79,7 @@ ENV YARA_VERSION "4.2.3" ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz" ENV YARA_RULES_SRC_DIR "/yara-rules-src" ENV YARA_RULES_DIR "/yara-rules" -ENV CAPA_VERSION "4.0.1" +ENV CAPA_VERSION "5.0.0" ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip" ENV CAPA_DIR "/opt/capa" ENV CAPA_BIN "${CAPA_DIR}/capa" diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index 2302bb606..ed4a1f224 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -9,7 +9,7 @@ ############################################################################### $fluent_bit_version = '2.0' -$fluent_bit_full_version = '2.0.6' +$fluent_bit_full_version = '2.0.9' ############################################################################### # select an item from a menu provided in an array From 7d0f63783fcf10952fb4eadf10928d425db276e7 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 13 Feb 2023 19:56:04 -0700 Subject: [PATCH 05/30] in packaged malcolm, symlink netbox-backup and netbox-restore --- malcolm-iso/build.sh | 2 ++ scripts/malcolm_appliance_packager.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 8805eb0a2..8671bc72a 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -131,6 +131,8 @@ if [ -d "$WORKDIR" ]; then pushd "$MALCOLM_DEST_DIR/scripts/" >/dev/null 2>&1 ln -s ./control.py auth_setup ln -s ./control.py logs + ln -s ./control.py netbox-backup + ln -s ./control.py netbox-restore ln -s ./control.py restart ln -s ./control.py start ln -s ./control.py status diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 84083b54c..53db37ca0 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -106,6 +106,8 @@ if mkdir "$DESTDIR"; then pushd "./scripts" >/dev/null 2>&1 ln -s ./control.py auth_setup ln -s ./control.py logs + ln -s ./control.py netbox-backup + ln -s ./control.py netbox-restore ln -s ./control.py restart ln -s ./control.py start ln -s ./control.py status From 438e7a27b3f360144b288398add3810820fb6204 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 14 Feb 2023 11:06:40 -0700 Subject: [PATCH 06/30] scan files with offline scanners by default --- scripts/install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index e0f4109ec..3d2c878f9 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -785,11 +785,11 @@ def tweak_malcolm_runtime( 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)' ) if fileCarveMode is not None: - if InstallerYesOrNo('Scan extracted files with ClamAV?', default=False): + if InstallerYesOrNo('Scan extracted files with ClamAV?', default=True): clamAvScan = True - if InstallerYesOrNo('Scan extracted files with Yara?', default=False): + if InstallerYesOrNo('Scan extracted files with Yara?', default=True): yaraScan = True - if InstallerYesOrNo('Scan extracted PE files with Capa?', default=False): + if InstallerYesOrNo('Scan extracted PE files with Capa?', default=True): capaScan = True if InstallerYesOrNo('Lookup extracted file hashes with VirusTotal?', default=False): while len(vtotApiKey) <= 1: From e7e4413f84fa8398c593fd5912cf48bc1d720584 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 15 Feb 2023 11:35:34 -0700 Subject: [PATCH 07/30] terminate netbox_init.py prior to restoring previous netbox database --- scripts/control.py | 16 +++++++--------- scripts/demo/reset_and_auto_populate.sh | 6 ++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/control.py b/scripts/control.py index 415e04eda..3c4dee9c0 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -63,6 +63,7 @@ def __exit__(self, *args): except: coloramaImported = False + ################################################################################################### # perform a service-keystore operation in a Docker container # @@ -103,7 +104,6 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs): uidGidDict = None try: - uidGidDict = GetUidGidFromComposeFile(args.composeFile) composeFileLines = list() @@ -213,7 +213,6 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs): raise Exception(f'Unable to identify docker image for {service} in {args.composeFile}') if dockerCmd is not None: - # append whatever other arguments to pass to the executable filespec if keystore_args: dockerCmd.extend(list(keystore_args)) @@ -326,7 +325,6 @@ def netboxRestore(backupFileName=None): global dockerComposeBin if backupFileName and os.path.isfile(backupFileName): - # docker-compose use local temporary path osEnv = os.environ.copy() osEnv['TMPDIR'] = MalcolmTmpPath @@ -345,6 +343,12 @@ def netboxRestore(backupFileName=None): f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}', ] + # if the netbox_init.py process is happening, interrupt it + dockerCmd = dockerCmdBase + ['netbox', 'bash', '-c', 'pgrep -f /usr/local/bin/netbox_init.py | xargs -r kill'] + err, results = run_process(dockerCmd, env=osEnv, debug=args.debug) + if (err != 0) and args.debug: + eprint(f'Error interrupting netbox_init.py: {results}') + # drop the existing netbox database dockerCmd = dockerCmdBase + ['netbox-postgres', 'dropdb', '-U', 'netbox', 'netbox', '--force'] err, results = run_process(dockerCmd, env=osEnv, debug=args.debug) @@ -510,7 +514,6 @@ def logs(): outputJson = LoadStrIfJson(messageStrToTestJson) if isinstance(outputJson, dict): - # if there's a timestamp, move it outside of the JSON to the beginning of the log string timeKey = None if 'time' in outputJson: @@ -531,7 +534,6 @@ def logs(): and ('job.position' in outputJson) and ('job.command' in outputJson) ): - # this is a status output line from supercronic, let's format and clean it up so it fits in better with the rest of the logs # remove some clutter for the display @@ -844,7 +846,6 @@ def authSetup(wipe=False): global opensslBin if YesOrNo('Store administrator username/password for local Malcolm access?', default=True): - # prompt username and password usernamePrevious = None password = None @@ -1023,7 +1024,6 @@ def authSetup(wipe=False): filebeatPath = os.path.join(MalcolmPath, os.path.join('filebeat', 'certs')) if YesOrNo('(Re)generate self-signed certificates for a remote log forwarder', default=True): with pushd(logstashPath): - # make clean to clean previous files for pat in ['*.srl', '*.csr', '*.key', '*.crt', '*.pem']: for oldfile in glob.glob(pat): @@ -1262,7 +1262,6 @@ def authSetup(wipe=False): 'Store username/password for email alert sender account? (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account)', default=False, ): - # prompt username and password emailPassword = None emailPasswordConfirm = None @@ -1539,7 +1538,6 @@ def main(): exit(2) with pushd(MalcolmPath): - # don't run this as root if (pyPlatform != PLATFORM_WINDOWS) and ( (os.getuid() == 0) or (os.geteuid() == 0) or (getpass.getuser() == 'root') diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 7b2c38113..9fea55074 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -274,6 +274,12 @@ if [[ -f "$MALCOLM_DOCKER_COMPOSE" ]] && \ if [[ -n "$NETBOX_BACKUP_FILE" ]] && [[ -f "$NETBOX_BACKUP_FILE" ]]; then # restore the netbox backup [[ -n $VERBOSE_FLAG ]] && echo "Restoring NetBox database backup" >&2 + # wait for NetBox to be ready with the initial startup before we go mucking around + until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | grep -q 'Unit configuration loaded successfully'; do + [[ -n $VERBOSE_FLAG ]] && echo "waiting for NetBox initialization to complete..." >&2 + sleep 10 + done + sleep 20 ./scripts/netbox-restore $VERBOSE_FLAG -f "$MALCOLM_FILE" --netbox-restore "$NETBOX_BACKUP_FILE" || true fi From a5089d6811dac354bb20e5fc9a2b39dbebf3f29f Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 15 Feb 2023 11:52:32 -0700 Subject: [PATCH 08/30] terminate netbox_init.py prior to restoring previous netbox database --- scripts/demo/reset_and_auto_populate.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 9fea55074..155239b8c 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -86,7 +86,8 @@ PCAP_RELATIVE_ADJUST="false" PCAP_PROCESS_PRE_WAIT=120 PCAP_PROCESS_IDLE_SECONDS=180 PCAP_PROCESS_IDLE_MAX_SECONDS=3600 -while getopts 'vwronlb:m:x:s:d:' OPTION; do +NETBOX_INIT_MAX_SECONDS=300 +while getopts 'vwronlb:m:i:x:s:d:' OPTION; do case "$OPTION" in v) VERBOSE_FLAG="-v" @@ -132,6 +133,12 @@ while getopts 'vwronlb:m:x:s:d:' OPTION; do fi ;; + i) + if [[ $OPTARG =~ $NUMERIC_REGEX ]] ; then + NETBOX_INIT_MAX_SECONDS=$OPTARG + fi + ;; + d) PCAP_DATE="$OPTARG" ;; @@ -275,9 +282,17 @@ if [[ -f "$MALCOLM_DOCKER_COMPOSE" ]] && \ # restore the netbox backup [[ -n $VERBOSE_FLAG ]] && echo "Restoring NetBox database backup" >&2 # wait for NetBox to be ready with the initial startup before we go mucking around - until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | grep -q 'Unit configuration loaded successfully'; do + CURRENT_TIME=$(date -u +%s) + FIRST_NETBOX_INIT_CHECK_TIME=$CURRENT_TIME + until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | tr -cd '\11\12\15\40-\176' | grep -q "Unit configuration loaded successfully"; do [[ -n $VERBOSE_FLAG ]] && echo "waiting for NetBox initialization to complete..." >&2 sleep 10 + # if it's been more than the maximum wait time, bail + CURRENT_TIME=$(date -u +%s) + if (( ($CURRENT_TIME - $FIRST_NETBOX_INIT_CHECK_TIME) >= $NETBOX_INIT_MAX_SECONDS )); then + [[ -n $VERBOSE_FLAG ]] && echo "Max wait time expired waiting for netbox_init" >&2 + break + fi done sleep 20 ./scripts/netbox-restore $VERBOSE_FLAG -f "$MALCOLM_FILE" --netbox-restore "$NETBOX_BACKUP_FILE" || true From e9af582dad7208551baec531c38186b4eeff3468 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 15 Feb 2023 11:55:21 -0700 Subject: [PATCH 09/30] terminate netbox_init.py prior to restoring previous netbox database --- scripts/demo/reset_and_auto_populate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 155239b8c..77fad7bc6 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -68,7 +68,6 @@ FULL_PWD="$($REALPATH "$(pwd)" | head -n 1)" # script options set -e set -u -set -o pipefail ENCODING="utf-8" # parse command-line options From e0f9f1e058537171e304e05f69ad8221dc04d4db Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 15 Feb 2023 11:57:45 -0700 Subject: [PATCH 10/30] terminate netbox_init.py prior to restoring previous netbox database --- scripts/demo/reset_and_auto_populate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 77fad7bc6..dd1997303 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -283,7 +283,7 @@ if [[ -f "$MALCOLM_DOCKER_COMPOSE" ]] && \ # wait for NetBox to be ready with the initial startup before we go mucking around CURRENT_TIME=$(date -u +%s) FIRST_NETBOX_INIT_CHECK_TIME=$CURRENT_TIME - until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | tr -cd '\11\12\15\40-\176' | grep -q "Unit configuration loaded successfully"; do + until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | grep -q "Unit configuration loaded successfully"; do [[ -n $VERBOSE_FLAG ]] && echo "waiting for NetBox initialization to complete..." >&2 sleep 10 # if it's been more than the maximum wait time, bail From 5fce7752a48ed0a674ce26d3380db7bd4f66fa83 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 16 Feb 2023 09:37:43 -0700 Subject: [PATCH 11/30] update beats to v8.6.2 --- Dockerfiles/filebeat.Dockerfile | 2 +- sensor-iso/beats/Dockerfile | 2 +- sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 92a6892b1..c11b0e1f5 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.6.1 +FROM docker.elastic.co/beats/filebeat-oss:8.6.2 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/beats/Dockerfile b/sensor-iso/beats/Dockerfile index 648bf6f18..5126c1bb3 100644 --- a/sensor-iso/beats/Dockerfile +++ b/sensor-iso/beats/Dockerfile @@ -41,7 +41,7 @@ RUN set -x && \ go run bootstrap.go ENV BEATS=filebeat -ENV BEATS_VERSION=8.6.1 +ENV BEATS_VERSION=8.6.2 ADD ./build.sh /build.sh RUN [ "chmod", "+x", "/build.sh" ] diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 6a821436d..603dd0ecb 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -20,7 +20,7 @@ export PATH="${ZEEK_DIR}"/bin:$PATH SURICATA_RULES_DIR="/etc/suricata/rules" -BEATS_VER="8.6.1" +BEATS_VER="8.6.2" BEATS_OSS="-oss" BEATS_DEB_URL_TEMPLATE_REPLACER="XXXXX" BEATS_DEB_URL_TEMPLATE="https://artifacts.elastic.co/downloads/beats/$BEATS_DEB_URL_TEMPLATE_REPLACER/$BEATS_DEB_URL_TEMPLATE_REPLACER$BEATS_OSS-$BEATS_VER-amd64.deb" From 0709fdc0cff079c8c64e42b9c9b962a38fd0bde0 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 16 Feb 2023 10:00:05 -0700 Subject: [PATCH 12/30] Bump werkzeug from 1.0.1 to 2.2.3 in /sensor-iso/interface --- sensor-iso/interface/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index 7742a82e3..b71e0ab60 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -13,4 +13,4 @@ python-dotenv==0.14.0 requests==2.26.0 six==1.15.0 urllib3==1.26.7 -Werkzeug==1.0.1 +Werkzeug==2.2.3 From 3c29c422b2ff630debcac6973921457a29440882 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 16 Feb 2023 10:43:46 -0700 Subject: [PATCH 13/30] update python package versions for sensor --- sensor-iso/interface/requirements.txt | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index b71e0ab60..ab47c4010 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -1,16 +1,16 @@ certifi==2022.12.7 -chardet==3.0.4 -click==7.1.2 -Flask==1.1.2 -Flask-Cors==3.0.9 -gunicorn==20.0.4 -idna==2.10 -itsdangerous==1.1.0 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -psutil -python-dotenv==0.14.0 -requests==2.26.0 -six==1.15.0 -urllib3==1.26.7 +chardet==5.1.0 +click==8.1.3 +Flask==2.2.3 +Flask-Cors==3.0.10 +gunicorn==20.1.0 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.2 +psutil==5.9.4 +python-dotenv==0.21.1 +requests==2.28.2 +six==1.16.0 +urllib3==1.26.14 Werkzeug==2.2.3 From 9627daa853349c03237462e0cb183d7673759250 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 22 Feb 2023 11:34:32 -0700 Subject: [PATCH 14/30] Update zeek to v5.0.7 (https://github.com/zeek/zeek/releases/tag/v5.0.7) --- Dockerfiles/zeek.Dockerfile | 2 +- sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 3df19c68b..d5dd08946 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -31,7 +31,7 @@ ENV PUSER_PRIV_DROP false # for download and install ARG ZEEK_LTS=true -ARG ZEEK_VERSION=5.0.6-0 +ARG ZEEK_VERSION=5.0.7-0 ENV ZEEK_LTS $ZEEK_LTS ENV ZEEK_VERSION $ZEEK_VERSION diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 603dd0ecb..af459e612 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -13,7 +13,7 @@ GITHUB_API_CURL_ARGS+=( -H ) GITHUB_API_CURL_ARGS+=( "Accept: application/vnd.github.v3+json" ) [[ -n "$GITHUB_TOKEN" ]] && GITHUB_API_CURL_ARGS+=( -H ) && GITHUB_API_CURL_ARGS+=( "Authorization: token $GITHUB_TOKEN" ) -ZEEK_VER=5.0.6-0 +ZEEK_VER=5.0.7-0 ZEEK_LTS=true ZEEK_DIR="/opt/zeek" export PATH="${ZEEK_DIR}"/bin:$PATH From abe8ffadfe8b25a5909ac02408c783cd3a5d4da7 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 27 Feb 2023 10:37:31 -0700 Subject: [PATCH 15/30] bump supercronic to v0.2.2 (https://github.com/aptible/supercronic/releases/tag/v0.2.2) --- Dockerfiles/dashboards-helper.Dockerfile | 4 ++-- Dockerfiles/file-monitor.Dockerfile | 4 ++-- Dockerfiles/filebeat.Dockerfile | 4 ++-- Dockerfiles/netbox.Dockerfile | 4 ++-- Dockerfiles/suricata.Dockerfile | 4 ++-- Dockerfiles/zeek.Dockerfile | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 529d8c5e7..3d56ce366 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -47,10 +47,10 @@ ENV DASHBOARDS_URL $DASHBOARDS_URL ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE ENV PATH="/data:${PATH}" -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV ECS_RELEASES_URL "https://api.github.com/repos/elastic/ecs/releases/latest" diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index b13613f72..ec3486357 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -89,10 +89,10 @@ ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index c11b0e1f5..ea6a808de 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -57,10 +57,10 @@ ARG FILEBEAT_TCP_PARSE_TARGET_FIELD="" ARG FILEBEAT_TCP_PARSE_DROP_FIELD="" ARG FILEBEAT_TCP_TAG="_malcolm_beats" -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV TINI_VERSION v0.19.0 diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index a9e842062..4349594f2 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -22,10 +22,10 @@ ENV PUSER "boxer" ENV PGROUP "boxer" ENV PUSER_PRIV_DROP true -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_DEVICETYPE_LIBRARY_URL "https://codeload.github.com/netbox-community/devicetype-library/tar.gz/master" diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 67c88890a..4dd34aead 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -27,10 +27,10 @@ ENV PGROUP "suricata" # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.24.2" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index d5dd08946..86ec72736 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -36,10 +36,10 @@ ARG ZEEK_VERSION=5.0.7-0 ENV ZEEK_LTS $ZEEK_LTS ENV ZEEK_VERSION $ZEEK_VERSION -ENV SUPERCRONIC_VERSION "0.2.1" +ENV SUPERCRONIC_VERSION "0.2.2" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70" +ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be" ENV SUPERCRONIC_CRONTAB "/etc/crontab" # for build From 23665c3c9433b61ebf80cc561aa1d76ef827a65a Mon Sep 17 00:00:00 2001 From: Ben Allen Date: Thu, 23 Feb 2023 18:19:56 -0500 Subject: [PATCH 16/30] Moved list of Malcolm URLs to when Malcolm is actually finished starting --- scripts/control.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/control.py b/scripts/control.py index 3c4dee9c0..77643137b 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -617,6 +617,23 @@ def logs(): else: time.sleep(0.5) + + if finishedStarting: + process.terminate() + # # TODO: Replace 'localhost' with an outwards-facing IP since I doubt anybody is + # accessing these from the Malcolm server. + print("Started Malcolm\n\n") + print("Malcolm services can be accessed via the following URLs:") + print("------------------------------------------------------------------------------") + print(" - Arkime: https://localhost/") + print(" - OpenSearch Dashboards: https://localhost/dashboards/") + print(" - PCAP upload (web): https://localhost/upload/") + print(" - PCAP upload (sftp): sftp://username@127.0.0.1:8022/files/") + print(" - Host and subnet name mapping editor: https://localhost/name-map-ui/") + print(" - NetBox: https://localhost/netbox/\n") + print(" - Account management: https://localhost:488/\n") + print(" - Documentation: https://localhost/readme/\n") + process.poll() @@ -820,19 +837,7 @@ def start(): # start docker err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'up', '--detach'], env=osEnv, debug=args.debug) - if err == 0: - eprint("Started Malcolm\n\n") - eprint("In a few minutes, Malcolm services will be accessible via the following URLs:") - eprint("------------------------------------------------------------------------------") - eprint(" - Arkime: https://localhost/") - eprint(" - OpenSearch Dashboards: https://localhost/dashboards/") - eprint(" - PCAP upload (web): https://localhost/upload/") - eprint(" - PCAP upload (sftp): sftp://username@127.0.0.1:8022/files/") - eprint(" - Host and subnet name mapping editor: https://localhost/name-map-ui/") - eprint(" - NetBox: https://localhost/netbox/\n") - eprint(" - Account management: https://localhost:488/\n") - eprint(" - Documentation: https://localhost/readme/\n") - else: + if err != 0: eprint("Malcolm failed to start\n") eprint("\n".join(out)) exit(err) From 2177413a1a81863b143cf159db25c8e0deba9a83 Mon Sep 17 00:00:00 2001 From: Ben Allen Date: Thu, 23 Feb 2023 18:01:22 -0500 Subject: [PATCH 17/30] Stop ./scripts/start once Malcolm has finished starting --- scripts/control.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/control.py b/scripts/control.py index 77643137b..44eefda62 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -460,6 +460,9 @@ def logs(): r'^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$' ) + finishedStartingRegEx = re.compile(r'.+Pipelines\s+running\s+\{.*:non_running_pipelines=>\[\]\}') + finishedStarting = False + # increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output osEnv = os.environ.copy() osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000' @@ -499,6 +502,8 @@ def logs(): outputStrEscaped = EscapeAnsi(outputStr) if ignoreRegEx.match(outputStrEscaped): pass ### print(f'!!!!!!!: {outputStr}') + elif finishedStartingRegEx.match(outputStrEscaped) and (args.cmdStart or args.cmdRestart): + finishedStarting = True else: serviceMatch = serviceRegEx.search(outputStrEscaped) serviceMatchFmt = serviceRegEx.search(outputStr) if coloramaImported else serviceMatch From 9248f78f6c591608edeeb65d34e3786b4f1dd675 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 27 Feb 2023 14:03:06 -0700 Subject: [PATCH 18/30] minor tweaks for the changes for start/restart exiting logs after startup is "finished". for cisagov/Malcolm#241 and cisagov/Malcolm#240. --- scripts/control.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/control.py b/scripts/control.py index 44eefda62..596ce0349 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -24,7 +24,7 @@ from malcolm_common import * from base64 import b64encode from collections import defaultdict, namedtuple -from subprocess import PIPE, STDOUT, Popen, check_call, CalledProcessError +from subprocess import PIPE, DEVNULL, Popen, TimeoutExpired from urllib.parse import urlparse try: @@ -492,6 +492,7 @@ def logs(): ][: 8 if args.service is not None else -1], env=osEnv, stdout=PIPE, + stderr=None if args.debug else DEVNULL, ) while True: output = process.stdout.readline() @@ -502,7 +503,11 @@ def logs(): outputStrEscaped = EscapeAnsi(outputStr) if ignoreRegEx.match(outputStrEscaped): pass ### print(f'!!!!!!!: {outputStr}') - elif finishedStartingRegEx.match(outputStrEscaped) and (args.cmdStart or args.cmdRestart): + elif ( + (args.cmdStart or args.cmdRestart) + and (not args.cmdLogs) + and finishedStartingRegEx.match(outputStrEscaped) + ): finishedStarting = True else: serviceMatch = serviceRegEx.search(outputStrEscaped) @@ -625,9 +630,13 @@ def logs(): if finishedStarting: process.terminate() + try: + process.wait(timeout=5.0) + except TimeoutExpired: + process.kill() # # TODO: Replace 'localhost' with an outwards-facing IP since I doubt anybody is # accessing these from the Malcolm server. - print("Started Malcolm\n\n") + print("\nStarted Malcolm\n\n") print("Malcolm services can be accessed via the following URLs:") print("------------------------------------------------------------------------------") print(" - Arkime: https://localhost/") From 69151eb199465dc963fd97c866a136033a3e767d Mon Sep 17 00:00:00 2001 From: cuifei Date: Tue, 28 Feb 2023 18:38:58 +0800 Subject: [PATCH 19/30] Modify a words --- shared/bin/pcap_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index f66571cf2..a55708785 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -449,7 +449,7 @@ def main(): else: preexistingDir = False if debug: - eprint(f'{scriptname}: creating "{args.baseDir}" to monitor') + eprint(f'{scriptName}: creating "{args.baseDir}" to monitor') pathlib.Path(args.baseDir).mkdir(parents=False, exist_ok=True) # if recursion was requested, get list of directories to monitor From d1e12751aeb689c6099e9d378bea0b7c2fcfb232 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 28 Feb 2023 15:17:40 -0700 Subject: [PATCH 20/30] change venv files to root ownership so that startup for non-1000 UIDs doesn't take so long --- Dockerfiles/netbox.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 4349594f2..3503b04c7 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -62,6 +62,8 @@ RUN apt-get -q update && \ usermod -a -G tty ${PUSER} && \ mkdir -p /opt/unit "${NETBOX_DEVICETYPE_LIBRARY_PATH}" && \ chown -R $PUSER:$PGROUP /etc/netbox /opt/unit /opt/netbox && \ + # trying to see if things still work if these are owned by root (to avoid a costly chown on container startup) + chown --silent -R root:root /opt/netbox/venv/* && \ cd "$(dirname "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" && \ curl -sSL "$NETBOX_DEVICETYPE_LIBRARY_URL" | tar xzvf - -C ./"$(basename "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" --strip-components 1 && \ mkdir -p /opt/netbox/netbox/$BASE_PATH && \ From ed990311574790249cfc1f582bb762390bbc4f06 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 28 Feb 2023 15:17:46 -0700 Subject: [PATCH 21/30] Update OpenSearch and OpenSearch Dashboards from v2.5.0 to v2.6.0 and Logstash from v8.4.0 to v8.6.1 - https://github.com/opensearch-project/OpenSearch/blob/bc50a2edcf29c3c41b7a777575c61e1874847d8a/release-notes/opensearch.release-notes-2.6.0.md - https://github.com/opensearch-project/OpenSearch-Dashboards/blob/69bcbfeea9bb345364e47f048cd5bcfe64c9c242/release-notes/opensearch-dashboards.release-notes-2.6.0.md - https://www.elastic.co/guide/en/logstash/current/releasenotes.html --- Dockerfiles/dashboards.Dockerfile | 8 ++++---- Dockerfiles/logstash.Dockerfile | 7 +++++-- Dockerfiles/opensearch.Dockerfile | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 1b469163f..0b70fcb96 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -14,10 +14,10 @@ ENV PGROUP "dashboarder" ENV TERM xterm -ARG OPENSEARCH_VERSION="2.5.0" +ARG OPENSEARCH_VERSION="2.6.0" ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION -ARG OPENSEARCH_DASHBOARDS_VERSION="2.5.0" +ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0" ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION # base system dependencies for checking out and building plugins @@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \ # runtime ################################################################## -FROM opensearchproject/opensearch-dashboards:2.5.0 +FROM opensearchproject/opensearch-dashboards:2.6.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -122,7 +122,7 @@ RUN yum upgrade -y && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \ + # TODO: when 2.6.0 is released /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \ # trying to see if things still work if these are owned by root (to avoid a costly chown on container startup) chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \ /usr/share/opensearch-dashboards/node_modules/* \ diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 7da33e796..ed4e4cbc5 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0 +FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.6.1 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -46,7 +46,8 @@ USER root ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini -RUN apt-get -q update && \ +RUN set -x && \ + apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get -y --no-install-recommends install \ gettext \ @@ -57,6 +58,8 @@ RUN apt-get -q update && \ tini && \ chmod +x /usr/bin/tini && \ pip3 install ipaddress supervisor manuf pyyaml && \ + export JAVA_HOME=/usr/share/logstash/jdk && \ + /usr/share/logstash/vendor/jruby/bin/jruby -S gem install bundler && \ echo "gem 'lru_cache'" >> /usr/share/logstash/Gemfile && \ /usr/share/logstash/bin/ruby -S bundle install && \ logstash-plugin install --preserve logstash-filter-translate logstash-filter-cidr logstash-filter-dns \ diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index 73019dfdc..d745a1705 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.5.0 +FROM opensearchproject/opensearch:2.6.0 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" From 19f6fa823a05bb74ec956afab9f9ce2616d4cac2 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 28 Feb 2023 21:34:54 -0700 Subject: [PATCH 22/30] Bump version to v23.03.0 --- docker-compose-standalone.yml | 46 +++++++++++++++++----------------- docker-compose.yml | 46 +++++++++++++++++----------------- docs/download.md | 4 +-- docs/hedgehog-iso-build.md | 2 +- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 40 ++++++++++++++--------------- docs/ubuntu-install-example.md | 40 ++++++++++++++--------------- 7 files changed, 90 insertions(+), 90 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 0826c93ec..ccdc723e3 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -359,7 +359,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: opensearch: - image: malcolmnetsec/opensearch:23.02.1 + image: malcolmnetsec/opensearch:23.03.0 restart: "no" stdin_open: false tty: true @@ -400,7 +400,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: malcolmnetsec/dashboards-helper:23.02.1 + image: malcolmnetsec/dashboards-helper:23.03.0 restart: "no" stdin_open: false tty: true @@ -431,7 +431,7 @@ services: retries: 3 start_period: 30s dashboards: - image: malcolmnetsec/dashboards:23.02.1 + image: malcolmnetsec/dashboards:23.03.0 restart: "no" stdin_open: false tty: true @@ -456,7 +456,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:23.02.1 + image: malcolmnetsec/logstash-oss:23.03.0 restart: "no" stdin_open: false tty: true @@ -499,7 +499,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:23.02.1 + image: malcolmnetsec/filebeat-oss:23.03.0 restart: "no" stdin_open: false tty: true @@ -538,7 +538,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:23.02.1 + image: malcolmnetsec/arkime:23.03.0 restart: "no" stdin_open: false tty: true @@ -576,7 +576,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:23.02.1 + image: malcolmnetsec/zeek:23.03.0 restart: "no" stdin_open: false tty: true @@ -615,7 +615,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: malcolmnetsec/zeek:23.02.1 + image: malcolmnetsec/zeek:23.03.0 restart: "no" stdin_open: false tty: true @@ -647,7 +647,7 @@ services: - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel suricata: - image: malcolmnetsec/suricata:23.02.1 + image: malcolmnetsec/suricata:23.03.0 restart: "no" stdin_open: false tty: true @@ -684,7 +684,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: malcolmnetsec/suricata:23.02.1 + image: malcolmnetsec/suricata:23.03.0 restart: "no" stdin_open: false tty: true @@ -711,7 +711,7 @@ services: - ./suricata-logs:/var/log/suricata - ./suricata/rules:/opt/suricata/rules:ro file-monitor: - image: malcolmnetsec/file-monitor:23.02.1 + image: malcolmnetsec/file-monitor:23.03.0 restart: "no" stdin_open: false tty: true @@ -735,7 +735,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:23.02.1 + image: malcolmnetsec/pcap-capture:23.03.0 restart: "no" stdin_open: false tty: true @@ -757,7 +757,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: malcolmnetsec/pcap-monitor:23.02.1 + image: malcolmnetsec/pcap-monitor:23.03.0 restart: "no" stdin_open: false tty: true @@ -783,7 +783,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:23.02.1 + image: malcolmnetsec/file-upload:23.03.0 restart: "no" stdin_open: false tty: true @@ -811,7 +811,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:23.02.1 + image: malcolmnetsec/htadmin:23.03.0 restart: "no" stdin_open: false tty: true @@ -835,7 +835,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:23.02.1 + image: malcolmnetsec/freq:23.03.0 restart: "no" stdin_open: false tty: true @@ -856,7 +856,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:23.02.1 + image: malcolmnetsec/name-map-ui:23.03.0 restart: "no" stdin_open: false tty: true @@ -877,7 +877,7 @@ services: retries: 3 start_period: 60s netbox: - image: malcolmnetsec/netbox:23.02.1 + image: malcolmnetsec/netbox:23.03.0 restart: "no" stdin_open: false tty: true @@ -908,7 +908,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: malcolmnetsec/postgresql:23.02.1 + image: malcolmnetsec/postgresql:23.03.0 restart: "no" stdin_open: false tty: true @@ -931,7 +931,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: malcolmnetsec/redis:23.02.1 + image: malcolmnetsec/redis:23.03.0 restart: "no" stdin_open: false tty: true @@ -958,7 +958,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: malcolmnetsec/redis:23.02.1 + image: malcolmnetsec/redis:23.03.0 restart: "no" stdin_open: false tty: true @@ -984,7 +984,7 @@ services: retries: 3 start_period: 45s api: - image: malcolmnetsec/api:23.02.1 + image: malcolmnetsec/api:23.03.0 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -1007,7 +1007,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:23.02.1 + image: malcolmnetsec/nginx-proxy:23.03.0 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index f305df50d..25cb415e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: malcolmnetsec/opensearch:23.02.1 + image: malcolmnetsec/opensearch:23.03.0 restart: "no" stdin_open: false tty: true @@ -406,7 +406,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: malcolmnetsec/dashboards-helper:23.02.1 + image: malcolmnetsec/dashboards-helper:23.03.0 restart: "no" stdin_open: false tty: true @@ -440,7 +440,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: malcolmnetsec/dashboards:23.02.1 + image: malcolmnetsec/dashboards:23.03.0 restart: "no" stdin_open: false tty: true @@ -468,7 +468,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:23.02.1 + image: malcolmnetsec/logstash-oss:23.03.0 restart: "no" stdin_open: false tty: true @@ -518,7 +518,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:23.02.1 + image: malcolmnetsec/filebeat-oss:23.03.0 restart: "no" stdin_open: false tty: true @@ -560,7 +560,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:23.02.1 + image: malcolmnetsec/arkime:23.03.0 restart: "no" stdin_open: false tty: true @@ -604,7 +604,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:23.02.1 + image: malcolmnetsec/zeek:23.03.0 restart: "no" stdin_open: false tty: true @@ -647,7 +647,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:23.02.1 + image: malcolmnetsec/zeek:23.03.0 restart: "no" stdin_open: false tty: true @@ -683,7 +683,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:23.02.1 + image: malcolmnetsec/suricata:23.03.0 restart: "no" stdin_open: false tty: true @@ -723,7 +723,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:23.02.1 + image: malcolmnetsec/suricata:23.03.0 restart: "no" stdin_open: false tty: true @@ -753,7 +753,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:23.02.1 + image: malcolmnetsec/file-monitor:23.03.0 restart: "no" stdin_open: false tty: true @@ -780,7 +780,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:23.02.1 + image: malcolmnetsec/pcap-capture:23.03.0 restart: "no" stdin_open: false tty: true @@ -805,7 +805,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:23.02.1 + image: malcolmnetsec/pcap-monitor:23.03.0 restart: "no" stdin_open: false tty: true @@ -834,7 +834,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:23.02.1 + image: malcolmnetsec/file-upload:23.03.0 restart: "no" stdin_open: false tty: true @@ -862,7 +862,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:23.02.1 + image: malcolmnetsec/htadmin:23.03.0 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -889,7 +889,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:23.02.1 + image: malcolmnetsec/freq:23.03.0 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -913,7 +913,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:23.02.1 + image: malcolmnetsec/name-map-ui:23.03.0 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -937,7 +937,7 @@ services: retries: 3 start_period: 60s netbox: - image: malcolmnetsec/netbox:23.02.1 + image: malcolmnetsec/netbox:23.03.0 build: context: . dockerfile: Dockerfiles/netbox.Dockerfile @@ -972,7 +972,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: malcolmnetsec/postgresql:23.02.1 + image: malcolmnetsec/postgresql:23.03.0 build: context: . dockerfile: Dockerfiles/postgresql.Dockerfile @@ -998,7 +998,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: malcolmnetsec/redis:23.02.1 + image: malcolmnetsec/redis:23.03.0 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -1028,7 +1028,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: malcolmnetsec/redis:23.02.1 + image: malcolmnetsec/redis:23.03.0 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -1057,7 +1057,7 @@ services: retries: 3 start_period: 45s api: - image: malcolmnetsec/api:23.02.1 + image: malcolmnetsec/api:23.03.0 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -1086,7 +1086,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:23.02.1 + image: malcolmnetsec/nginx-proxy:23.03.0 restart: "no" stdin_open: false tty: true diff --git a/docs/download.md b/docs/download.md index b0b2acb21..b4d63b794 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.02.1.iso](/iso/malcolm-23.02.1.iso) (5.3GiB) | [`xxxxxxxx`](/iso/malcolm-23.02.1.iso.sha256.txt) | +| [malcolm-23.03.0.iso](/iso/malcolm-23.03.0.iso) (5.3GiB) | [`xxxxxxxx`](/iso/malcolm-23.03.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.02.1.iso](/iso/hedgehog-23.02.1.iso) (2.3GiB) | [`xxxxxxxx`](/iso/hedgehog-23.02.1.iso.sha256.txt) | +| [hedgehog-23.03.0.iso](/iso/hedgehog-23.03.0.iso) (2.3GiB) | [`xxxxxxxx`](/iso/hedgehog-23.03.0.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 8f0167a7f..c37ed0b23 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.02.1.iso" +Finished, created "/sensor-build/hedgehog-23.03.0.iso" … ``` diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 7df41506d..edeb32810 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.02.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.03.0.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 5cedaaa8b..01dbc0a35 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -53,26 +53,26 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 23.02.1 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 23.02.1 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 23.02.1 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 23.02.1 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/file-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 23.02.1 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/filebeat-oss 23.02.1 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/freq 23.02.1 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 23.02.1 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 23.02.1 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 23.02.1 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/netbox 23.02.1 xxxxxxxxxxxx 3 days ago 1.01GB -malcolmnetsec/nginx-proxy 23.02.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 23.02.1 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 23.02.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/postgresql 23.02.1 xxxxxxxxxxxx 3 days ago 268MB -malcolmnetsec/redis 23.02.1 xxxxxxxxxxxx 3 days ago 34.2MB -malcolmnetsec/suricata 23.02.1 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 23.02.1 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 23.03.0 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 23.03.0 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 23.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 23.03.0 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/file-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 23.03.0 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/filebeat-oss 23.03.0 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/freq 23.03.0 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 23.03.0 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 23.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 23.03.0 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/netbox 23.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +malcolmnetsec/nginx-proxy 23.03.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 23.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 23.03.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/postgresql 23.03.0 xxxxxxxxxxxx 3 days ago 268MB +malcolmnetsec/redis 23.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +malcolmnetsec/suricata 23.03.0 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 23.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 5850b6d33..61785dcde 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -261,26 +261,26 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 23.02.1 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 23.02.1 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 23.02.1 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 23.02.1 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/file-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 23.02.1 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/filebeat-oss 23.02.1 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/freq 23.02.1 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 23.02.1 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 23.02.1 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 23.02.1 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/netbox 23.02.1 xxxxxxxxxxxx 3 days ago 1.01GB -malcolmnetsec/nginx-proxy 23.02.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 23.02.1 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 23.02.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 23.02.1 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/postgresql 23.02.1 xxxxxxxxxxxx 3 days ago 268MB -malcolmnetsec/redis 23.02.1 xxxxxxxxxxxx 3 days ago 34.2MB -malcolmnetsec/suricata 23.02.1 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 23.02.1 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 23.03.0 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 23.03.0 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 23.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 23.03.0 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/file-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 23.03.0 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/filebeat-oss 23.03.0 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/freq 23.03.0 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 23.03.0 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 23.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 23.03.0 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/netbox 23.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +malcolmnetsec/nginx-proxy 23.03.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 23.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 23.03.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/postgresql 23.03.0 xxxxxxxxxxxx 3 days ago 268MB +malcolmnetsec/redis 23.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +malcolmnetsec/suricata 23.03.0 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 23.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. From bc424c86bd1fb2df8d154513d0d31cae20208845 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 1 Mar 2023 10:58:03 -0700 Subject: [PATCH 23/30] Bump Arkime to v4.2.0 (https://github.com/arkime/arkime/blob/6d549e51ddded0d303da07acc8c9ac3af5f405a7/CHANGELOG#L33-L60) --- Dockerfiles/arkime.Dockerfile | 2 +- sensor-iso/arkime/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index f5c6f839b..889b703cb 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "v4.1.0" +ENV ARKIME_VERSION "v4.2.0" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index df5b76424..32b8f1bb8 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "4.1.0" +ENV ARKIME_VERSION "4.2.0" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \ From 6317045194e65a6106d6a4db927721a499aafb5c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 1 Mar 2023 16:40:50 -0700 Subject: [PATCH 24/30] update opensearch-py to v2.2.0 (https://github.com/opensearch-project/opensearch-py/releases/tag/v2.2.0) (still needs testing) --- Dockerfiles/pcap-monitor.Dockerfile | 2 +- api/project/__init__.py | 46 ++++++++++++++++++----------- api/requirements.txt | 3 +- shared/bin/pcap_watcher.py | 38 +++++++++++------------- 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index ee19f218b..7abeb18df 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -59,7 +59,7 @@ RUN apt-get -q update && \ vim-tiny && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - pip3 install --no-cache-dir opensearch-py opensearch-dsl pyzmq pyinotify python-magic requests && \ + pip3 install --no-cache-dir opensearch-py pyzmq pyinotify python-magic requests && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/api/project/__init__.py b/api/project/__init__.py index b18119bc7..df906f6f6 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -1,7 +1,6 @@ import dateparser import json import malcolm_common -import opensearch_dsl import opensearchpy import os import pytz @@ -167,15 +166,15 @@ else defaultdict(lambda: None) ) if opensearchCreds['user'] is not None: - opensearchDslHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}" + opensearchHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}" opensearchReqHttpAuth = HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password']) else: - opensearchDslHttpAuth = None + opensearchHttpAuth = None opensearchReqHttpAuth = None -opensearch_dsl.connections.create_connection( +opensearchClient = opensearchpy.OpenSearch( hosts=[opensearchUrl], - http_auth=opensearchDslHttpAuth, + http_auth=opensearchHttpAuth, verify_certs=opensearchSslVerify, ssl_assert_hostname=False, ssl_show_warn=False, @@ -333,7 +332,7 @@ def filtertime(search, args, default_from="1 day ago", default_to="now"): Parameters ---------- - search : opensearch_dsl.Search + search : opensearchpy.Search The object representing the OpenSearch Search query args : dict The dictionary which should contain 'from' and 'to' times (see gettimes) @@ -377,7 +376,7 @@ def filtervalues(search, args): Parameters ---------- - search : opensearch_dsl.Search + search : opensearchpy.Search The object representing the OpenSearch Search query args : dict The dictionary which should contain 'filter' (see getfilters) @@ -413,7 +412,7 @@ def filtervalues(search, args): ) else: # field does not exist ("is null") - s = s.filter('bool', must_not=opensearch_dsl.Q('exists', field=fieldname)) + s = s.filter('bool', must_not=opensearchpy.helpers.query.Q('exists', field=fieldname)) if debugApi: print(f'filtervalues: {json.dumps(s.to_dict())}') @@ -442,8 +441,11 @@ def bucketfield(fieldname, current_request, urls=None): fields the name of the field(s) on which the aggregation was performed """ - s = opensearch_dsl.Search( - using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_INDEX_PATTERN"] + global opensearchClient + + s = opensearchpy.Search( + using=opensearchClient, + index=app.config["ARKIME_INDEX_PATTERN"], ).extra(size=0) args = get_request_arguments(current_request) start_time_ms, end_time_ms, s = filtertime(s, args) @@ -523,10 +525,13 @@ def document(index): results array of the documents retrieved (up to 'limit') """ + global opensearchClient + args = get_request_arguments(request) - s = opensearch_dsl.Search(using=opensearch_dsl.connections.get_connection(), index=index).extra( - size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"])) - ) + s = opensearchpy.Search( + using=opensearchClient, + index=index, + ).extra(size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"]))) start_time_ms, end_time_ms, s = filtertime(s, args, default_from="1970-1-1", default_to="now") filters, s = filtervalues(s, args) return jsonify( @@ -574,6 +579,8 @@ def fields(): fields A dict of dicts where key is the field name and value may contain 'description' and 'type' """ + global opensearchClient + args = get_request_arguments(request) templateName = args['template'] if 'template' in args else app.config["MALCOLM_TEMPLATE"] @@ -585,8 +592,9 @@ def fields(): if arkimeFields: try: # get fields from Arkime's field's table - s = opensearch_dsl.Search( - using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_FIELDS_INDEX"] + s = opensearchpy.Search( + using=opensearchClient, + index=app.config["ARKIME_FIELDS_INDEX"], ).extra(size=5000) for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]: if (fieldname := deep_get(hit, ['dbField2'])) and (fieldname not in fields): @@ -697,6 +705,8 @@ def version(): opensearch_health a JSON structure containing OpenSearch cluster health """ + global opensearchClient + return jsonify( version=app.config["MALCOLM_VERSION"], built=app.config["BUILD_DATE"], @@ -706,7 +716,7 @@ def version(): auth=opensearchReqHttpAuth, verify=opensearchSslVerify, ).json(), - opensearch_health=opensearch_dsl.connections.get_connection().cluster.health(), + opensearch_health=opensearchClient.cluster.health(), ) @@ -783,6 +793,8 @@ def event(): status the JSON-formatted OpenSearch response from indexing/updating the alert record """ + global opensearchClient + alert = {} idxResponse = {} data = get_request_arguments(request) @@ -880,7 +892,7 @@ def event(): alert['event']['hits'] = hitCount docDateStr = dateparser.parse(alert['@timestamp']).strftime('%y%m%d') - idxResponse = opensearch_dsl.connections.get_connection().index( + idxResponse = opensearchClient.index( index=f"{app.config['ARKIME_INDEX_PATTERN'].rstrip('*')}{docDateStr}", id=f"{docDateStr}-{alert['event']['id']}", body=alert, diff --git a/api/requirements.txt b/api/requirements.txt index 3c3b90f4a..77c4ee103 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,8 +1,7 @@ pytz==2021.3 Flask==2.0.2 gunicorn==20.1.0 -opensearch-py==2.1.1 -opensearch-dsl==2.0.1 +opensearch-py==2.2.0 requests==2.26.0 regex==2022.3.2 dateparser==1.1.1 \ No newline at end of file diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index a55708785..132c9f5f0 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -26,8 +26,7 @@ from pcap_utils import * from collections import defaultdict -import opensearchpy -import opensearch_dsl +from opensearchpy import OpenSearch, Search ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -44,33 +43,33 @@ verboseDebug = False pdbFlagged = False args = None -opensearchDslHttpAuth = None +opensearchHttpAuth = None scriptName = os.path.basename(__file__) scriptPath = os.path.dirname(os.path.realpath(__file__)) origPath = os.getcwd() shuttingDown = False DEFAULT_NODE_NAME = os.getenv('PCAP_NODE_NAME', 'malcolm') + ################################################################################################### # watch files written to and moved to this directory class EventWatcher(pyinotify.ProcessEvent): - # notify on files written in-place then closed (IN_CLOSE_WRITE), and moved into this directory (IN_MOVED_TO) _methods = ["IN_CLOSE_WRITE", "IN_MOVED_TO"] def __init__(self): global args - global opensearchDslHttpAuth + global opensearchHttpAuth global debug global verboseDebug super().__init__() self.useOpenSearch = False + self.openSearchClient = None # if we're going to be querying OpenSearch for past PCAP file status, connect now if args.opensearchUrl is not None: - connected = False healthy = False @@ -79,16 +78,18 @@ def __init__(self): try: if debug: eprint(f"{scriptName}:\tconnecting to OpenSearch {args.opensearchUrl}...") - opensearch_dsl.connections.create_connection( + + openSearchClient = OpenSearch( hosts=[args.opensearchUrl], - http_auth=opensearchDslHttpAuth, + http_auth=opensearchHttpAuth, verify_certs=args.opensearchSslVerify, ssl_assert_hostname=False, ssl_show_warn=False, ) + if verboseDebug: - eprint(f"{scriptName}:\t{opensearch_dsl.connections.get_connection().cluster.health()}") - connected = opensearch_dsl.connections.get_connection() is not None + eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") + connected = openSearchClient is not None except opensearchpy.exceptions.ConnectionError as connError: if debug: @@ -104,11 +105,12 @@ def __init__(self): try: if debug: eprint(f"{scriptName}:\twaiting for OpenSearch to be healthy") - opensearch_dsl.connections.get_connection().cluster.health( - index=ARKIME_FILES_INDEX, wait_for_status='yellow' + openSearchClient.cluster.health( + index=ARKIME_FILES_INDEX, + wait_for_status='yellow', ) if verboseDebug: - eprint(f"{scriptName}:\t{opensearch_dsl.connections.get_connection().cluster.health()}") + eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") healthy = True except opensearchpy.exceptions.ConnectionTimeout as connError: @@ -140,10 +142,8 @@ def __init__(self): ################################################################################################### # set up event processor to append processed events from to the event queue def event_process_generator(cls, method): - # actual method called when we are notified of a file def _method_name(self, event): - global args global debug global verboseDebug @@ -153,7 +153,6 @@ def _method_name(self, event): # the entity must be a regular PCAP file and actually exist if (not event.dir) and os.path.isfile(event.pathname): - # get the file magic description and mime type fileMime = magic.from_file(event.pathname, mime=True) fileType = magic.from_file(event.pathname) @@ -163,14 +162,13 @@ def _method_name(self, event): if (args.minBytes <= fileSize <= args.maxBytes) and ( (fileMime in PCAP_MIME_TYPES) or ('pcap-ng' in fileType) ): - relativePath = remove_prefix(event.pathname, os.path.join(args.baseDir, '')) # check with Arkime's files index in OpenSearch and make sure it's not a duplicate fileIsDuplicate = False if self.useOpenSearch: s = ( - opensearch_dsl.Search(index=ARKIME_FILES_INDEX) + Search(using=self.openSearchClient, index=ARKIME_FILES_INDEX) .filter("term", node=args.nodeName) .query("wildcard", name=f"*{os.path.sep}{relativePath}") ) @@ -243,7 +241,7 @@ def debug_toggle_handler(signum, frame): # main def main(): global args - global opensearchDslHttpAuth + global opensearchHttpAuth global debug global verboseDebug global debugToggled @@ -423,7 +421,7 @@ def main(): args.opensearchUrl = 'http://opensearch:9200' elif 'url' in opensearchCreds: args.opensearchUrl = opensearchCreds['url'] - opensearchDslHttpAuth = ( + opensearchHttpAuth = ( f"{opensearchCreds['user']}:{opensearchCreds['password']}" if opensearchCreds['user'] is not None else None ) From ebfa54e7002915471c97ed8c6b95e0b1af62ff26 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 2 Mar 2023 16:22:52 -0700 Subject: [PATCH 25/30] Fix issue with new opensearch-py --- shared/bin/pcap_watcher.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 132c9f5f0..6688c5431 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -27,6 +27,7 @@ from collections import defaultdict from opensearchpy import OpenSearch, Search +from opensearchpy.exceptions import ConnectionError, ConnectionTimeout ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -91,7 +92,7 @@ def __init__(self): eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") connected = openSearchClient is not None - except opensearchpy.exceptions.ConnectionError as connError: + except ConnectionError as connError: if debug: eprint(f"{scriptName}:\tOpenSearch connection error: {connError}") @@ -113,7 +114,7 @@ def __init__(self): eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") healthy = True - except opensearchpy.exceptions.ConnectionTimeout as connError: + except ConnectionTimeout as connError: if verboseDebug: eprint(f"{scriptName}:\tOpenSearch health check: {connError}") From fda6c98e4495e2af46d9f90100d20677ac184a8b Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 7 Mar 2023 12:57:12 -0700 Subject: [PATCH 26/30] Added a 'configure malcolm' menu item --- .../xfce4/panel/launcher-29/16782182331.desktop | 10 ++++++++++ .../xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml | 7 +++++++ .../usr/share/applications/malcolm-configure.desktop | 9 +++++++++ 3 files changed, 26 insertions(+) create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop create mode 100644 malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop new file mode 100644 index 000000000..91947e698 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop @@ -0,0 +1,10 @@ +[Desktop Entry] +Name=Configure Malcolm +Exec=tilix --title="Malcolm Configuration" --maximize -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/install.py --configure" +Comment=install.py --configure +Terminal=false +Type=Application +Icon=org.xfce.settings.manager +Categories=Network; +StartupNotify=true +X-XFCE-Source=file:///usr/share/applications/malcolm-configure.desktop \ No newline at end of file diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml index 6c0a032a4..9d0f3e6b6 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml @@ -22,6 +22,7 @@ + @@ -63,6 +64,7 @@ + @@ -150,5 +152,10 @@ + + + + + diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop new file mode 100644 index 000000000..253286161 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop @@ -0,0 +1,9 @@ +[Desktop Entry] +Name=Configure Malcolm +Exec=tilix --title="Malcolm Configuration" --maximize -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/install.py --configure" +Comment=install.py --configure +Terminal=false +Type=Application +Icon=org.xfce.settings.manager +Categories=Network; +StartupNotify=true \ No newline at end of file From a4506eb26039278fa555f9506ca0848ee3d27b81 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 7 Mar 2023 14:21:36 -0700 Subject: [PATCH 27/30] added better reconnect/checking in pcap_watcher.py --- shared/bin/pcap_watcher.py | 66 +++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 6688c5431..34846d2dd 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -28,6 +28,7 @@ from opensearchpy import OpenSearch, Search from opensearchpy.exceptions import ConnectionError, ConnectionTimeout +from urllib3.exceptions import NewConnectionError ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -77,28 +78,52 @@ def __init__(self): # create the connection to OpenSearch while (not connected) and (not shuttingDown): try: - if debug: - eprint(f"{scriptName}:\tconnecting to OpenSearch {args.opensearchUrl}...") - - openSearchClient = OpenSearch( - hosts=[args.opensearchUrl], - http_auth=opensearchHttpAuth, - verify_certs=args.opensearchSslVerify, - ssl_assert_hostname=False, - ssl_show_warn=False, - ) + try: + if debug: + eprint(f"{scriptName}:\tconnecting to OpenSearch {args.opensearchUrl}...") - if verboseDebug: - eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") - connected = openSearchClient is not None + self.openSearchClient = OpenSearch( + hosts=[args.opensearchUrl], + http_auth=opensearchHttpAuth, + verify_certs=args.opensearchSslVerify, + ssl_assert_hostname=False, + ssl_show_warn=False, + request_timeout=1, + ) + + if verboseDebug: + eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}") + + self.openSearchClient.cluster.health( + wait_for_status='red', + request_timeout=1, + ) + + if verboseDebug: + eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}") + + connected = self.openSearchClient is not None + if not connected: + time.sleep(1) + + except ( + ConnectionError, + ConnectionTimeout, + ConnectionRefusedError, + NewConnectionError, + ) as connError: + if debug: + eprint(f"{scriptName}:\tOpenSearch connection error: {connError}") - except ConnectionError as connError: + except Exception as genericError: if debug: - eprint(f"{scriptName}:\tOpenSearch connection error: {connError}") + eprint(f"{scriptName}:\tUnexpected exception while connecting to OpenSearch: {genericError}") if (not connected) and args.opensearchWaitForHealth: time.sleep(1) else: + if args.opensearchWaitForHealth: + time.sleep(1) break # if requested, wait for at least "yellow" health in the cluster for the "files" index @@ -106,15 +131,20 @@ def __init__(self): try: if debug: eprint(f"{scriptName}:\twaiting for OpenSearch to be healthy") - openSearchClient.cluster.health( + self.openSearchClient.cluster.health( index=ARKIME_FILES_INDEX, wait_for_status='yellow', ) if verboseDebug: - eprint(f"{scriptName}:\t{openSearchClient.cluster.health()}") + eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}") healthy = True - except ConnectionTimeout as connError: + except ( + ConnectionError, + ConnectionTimeout, + ConnectionRefusedError, + NewConnectionError, + ) as connError: if verboseDebug: eprint(f"{scriptName}:\tOpenSearch health check: {connError}") From 3084dcb9ec528430328029485c2e41c006fabff7 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 7 Mar 2023 20:47:29 -0700 Subject: [PATCH 28/30] minor usability tweaks for ISO --- .../.config/autostart/set-malcolm-gtk-bookmark.desktop | 7 +++++++ .../includes.chroot/etc/skel/.config/gtk-3.0/bookmarks | 1 + .../.config/xfce4/panel/launcher-15/16346759461.desktop | 6 +++--- shared/bin/set-malcolm-gtk-bookmark.sh | 8 ++++++++ 4 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop create mode 100644 sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks create mode 100755 shared/bin/set-malcolm-gtk-bookmark.sh diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop new file mode 100644 index 000000000..0f31f1ae1 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Encoding=UTF-8 +Name=set-malcolm-gtk-bookmark +Comment=Add Malcolm directory to GTK-3.0 bookmarks +Exec=/usr/local/bin/set-malcolm-gtk-bookmark.sh +Terminal=false +Type=Application diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks b/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks new file mode 100644 index 000000000..5b6b943a6 --- /dev/null +++ b/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks @@ -0,0 +1 @@ +file:///opt/sensor/sensor_ctl \ No newline at end of file diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop index fcaabc15c..8210ef836 100644 --- a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop +++ b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop @@ -3,7 +3,7 @@ Version=1.0 Name=Tilix Comment=A tiling terminal for Gnome Keywords=shell;prompt;command;commandline;cmd; -Exec=tilix +Exec=tilix --working-directory=/opt/sensor/sensor_ctl Terminal=false Type=Application StartupNotify=true @@ -15,11 +15,11 @@ X-XFCE-Source=file:///usr/share/applications/com.gexperts.Tilix.desktop [Desktop Action new-window] Name=New Window -Exec=tilix --action=app-new-window +Exec=tilix --action=app-new-window --working-directory=/opt/sensor/sensor_ctl [Desktop Action new-session] Name=New Session -Exec=tilix --action=app-new-session +Exec=tilix --action=app-new-session --working-directory=/opt/sensor/sensor_ctl [Desktop Action preferences] Name=Preferences diff --git a/shared/bin/set-malcolm-gtk-bookmark.sh b/shared/bin/set-malcolm-gtk-bookmark.sh new file mode 100755 index 000000000..f324b94dc --- /dev/null +++ b/shared/bin/set-malcolm-gtk-bookmark.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. + +if ! grep -q Malcolm$ "$HOME"/.config/gtk-3.0/bookmarks && [[ -d "$HOME"/Malcolm ]]; then + mkdir -p "$HOME"/.config/gtk-3.0/ + echo -e "\nfile://$HOME/Malcolm" >> "$HOME"/.config/gtk-3.0/bookmarks +fi From 43e63a4470b11afda10907321ca4a861b11be54b Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 7 Mar 2023 20:48:47 -0700 Subject: [PATCH 29/30] (hopefully) fix idaholab/Malcolm#151, last few seconds' Zeek logs prior to log rotation may be lost --- logstash/pipelines/zeek/10_zeek_prep.conf | 16 ++++++++++------ logstash/pipelines/zeek/11_zeek_parse.conf | 10 ---------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 7f92bbeb2..15d1d17fa 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -6,14 +6,18 @@ filter { drop { id => "drop_zeek_invalid_logs" } } - # tags may have been specified, like: conn(tagA,tagB,tagC).log, extract the log type (conn) and the tags (tagA,tagB,tagC) - # also normalize log types with - in their names to _ (e.g., opcua-binary -> opcua_binary) + # - Tags may have been specified, like: conn(tagA,tagB,tagC).log, extract the log type (conn) and the tags (tagA,tagB,tagC). + # - Normalize log types with - in their names to _ (e.g., opcua-binary -> opcua_binary). + # - Zeek log files might be caught by filebeat right in the middle of being renamed/moved as + # part of log rotation (ie., renamed from conn.log to conn.2020-01-16-14-00-00.log or + # conn.2020_01_16_14_00_00.log). We don't care about that, ignore the date part and just process + # the log source as we normally would. ruby { - id => "ruby_zeek_source_extract" - #↓Type ↓Tags + id => "ruby_zeek_log_source_extract" + # ↓Type ↓Tags ↓Rotate Timestamp (discard) ↓.log (discard) code => " - if fileParts = event.get('[log][file][path]').split('/').last.match(/^(.*?)(?:\((.*)\))?\.log/i) then - logType, tags = fileParts.captures + if fileParts = event.get('[log][file][path]').split('/').last.match(/^(.*?)(?:\((.*)\))?(?:\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2})?\.log/i) then + logType, tags = fileParts.captures event.set('[log_source]', logType.gsub('-', '_')) unless logType.nil? event.set('[@metadata][zeek_log_tags]', tags) unless tags.nil? end" diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index e720c3afb..9314d04b9 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -5077,10 +5077,6 @@ filter { add_tag => [ "ics" ] } - } else if ([log_source] =~ /\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}$/) { - # filebeat caught a file right in the middle of being renamed/moved - drop { id => "drop_renamed_logfile_opcua" } - } else { # some other unknown zeek opcua- log file. should start with ts at least! csv { @@ -5105,12 +5101,6 @@ filter { } # if / else if for opcua log types - } else if ([log_source] =~ /\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}$/) { - # filebeat caught a file right in the middle of being renamed/moved - # (ie., renamed from conn.log to conn.2020-01-16-14-00-00.log or conn.2020_01_16_14_00_00.log). - # this has actually already been processed, so ignore this event. - drop { id => "drop_renamed_logfile" } - } else { # some other unknown zeek log file. should start with ts at least! csv { From a9da1e1f83f9a9ba6773017e2bb56d5f46f047f2 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 8 Mar 2023 08:24:43 -0700 Subject: [PATCH 30/30] some documentation updates prior to release --- _config.yml | 2 +- docs/download.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_config.yml b/_config.yml index 68a31147b..0e92a8819 100644 --- a/_config.yml +++ b/_config.yml @@ -4,7 +4,7 @@ description: A powerful, easily deployable network traffic analysis tool suite logo: docs/images/logo/Malcolm_outline_banner_dark.png remote_theme: pages-themes/minimal@v0.2.0 external_download_url: https://malcolm.fyi/docs/download.html -youtube_url: https://www.youtube.com/c/MalcolmNetworkTrafficAnalysisToolSuite +youtube_url: https://www.youtube.com/@MalcolmNetworkTrafficAnalysis mastodon: id: malcolm@malcolm.fyi url: https://infosec.exchange/@mmguero diff --git a/docs/download.md b/docs/download.md index b4d63b794..f0ae7097e 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.03.0.iso](/iso/malcolm-23.03.0.iso) (5.3GiB) | [`xxxxxxxx`](/iso/malcolm-23.03.0.iso.sha256.txt) | +| [malcolm-23.03.0.iso](/iso/malcolm-23.03.0.iso) (5.3GiB) | [`9459fb0ce61fba8c7a9a9457b24d42182519dbb62247111471e38f8c190113eb`](/iso/malcolm-23.03.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.03.0.iso](/iso/hedgehog-23.03.0.iso) (2.3GiB) | [`xxxxxxxx`](/iso/hedgehog-23.03.0.iso.sha256.txt) | +| [hedgehog-23.03.0.iso](/iso/hedgehog-23.03.0.iso) (2.3GiB) | [`3cdba91e417f6ada83130aabc3be38dd0a8b12b6bda227859a546ace198680bc`](/iso/hedgehog-23.03.0.iso.sha256.txt) | ## Warning