#169, make htadmin work with an /auth URI instead of a separate :488 …

…port

Dockerfiles/htadmin.Dockerfile

@@ -61,6 +61,8 @@ RUN apt-get -q update && \
   cd /tmp && \
     mkdir -p ./htadmin && \
     curl -sSL "$HTADMIN_URL" | tar xzvf - -C ./htadmin --strip-components 1 && \
+    find /tmp/htadmin -type f -name index.php -execdir mv index.php htadmin.php \; && \
+    find /tmp/htadmin -type f -exec sed -i 's/index.php/htadmin.php/g' "{}" \; && \
     mv /tmp/htadmin/sites/html/htadmin /var/www/htadmin && \
     cd /var/www/htadmin && \
     ( grep -rhoPi "(src|href)=['\"]https?://.+?['\"]" ./includes/* | sed "s/^[a-zA-Z]*=['\"]*//" | sed "s/['\"]$//" | xargs -r -l curl -s -S -L -J -O ) && \

Dockerfiles/nginx.Dockerfile

@@ -87,10 +87,12 @@ ENV NGINX_LDAP_TLS_STUNNEL_CHECK_IP $NGINX_LDAP_TLS_STUNNEL_CHECK_IP
 ENV NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL
 
 # build latest nginx with nginx-auth-ldap
-ENV NGINX_VERSION=1.20.2
+ENV NGINX_VERSION=1.22.1
 ENV NGINX_AUTH_LDAP_BRANCH=master
+ENV NGINX_HTTP_SUB_FILTER_BRANCH=master
 
 ADD https://codeload.github.com/mmguero-dev/nginx-auth-ldap/tar.gz/$NGINX_AUTH_LDAP_BRANCH /nginx-auth-ldap.tar.gz
+ADD https://codeload.github.com/yaoweibin/ngx_http_substitutions_filter_module/tar.gz/$NGINX_HTTP_SUB_FILTER_BRANCH /ngx_http_substitutions_filter_module-master.tar.gz
 ADD http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz /nginx.tar.gz
 
 RUN set -x ; \
@@ -140,6 +142,7 @@ RUN set -x ; \
     --with-file-aio \
     --with-http_v2_module \
     --add-module=/usr/src/nginx-auth-ldap \
+    --add-module=/usr/src/ngx_http_substitutions_filter_module \
   " ; \
   apk update --no-cache; \
   apk upgrade --no-cache; \
@@ -166,9 +169,10 @@ RUN set -x ; \
     zlib-dev \
     ; \
     \
-  mkdir -p /usr/src/nginx-auth-ldap /www /www/logs/nginx ; \
+  mkdir -p /usr/src/nginx-auth-ldap /usr/src/ngx_http_substitutions_filter_module /www /www/logs/nginx ; \
   tar -zxC /usr/src -f /nginx.tar.gz ; \
   tar -zxC /usr/src/nginx-auth-ldap --strip=1 -f /nginx-auth-ldap.tar.gz ; \
+  tar -zxC /usr/src/ngx_http_substitutions_filter_module --strip=1 -f /ngx_http_substitutions_filter_module-master.tar.gz ; \
   cd /usr/src/nginx-$NGINX_VERSION ; \
   ./configure $CONFIG --with-debug ; \
   make -j$(getconf _NPROCESSORS_ONLN) ; \
@@ -216,7 +220,7 @@ RUN set -x ; \
   apk del .nginx-build-deps ; \
   apk del .gettext ; \
   mv /tmp/envsubst /usr/local/bin/ ; \
-  rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /nginx.tar.gz /nginx-auth-ldap.tar.gz; \
+  rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /nginx.tar.gz /nginx-auth-ldap.tar.gz /ngx_http_substitutions_filter_module-master.tar.gz; \
   touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf;
 
 COPY --from=jwilder/nginx-proxy:alpine /app/nginx.tmpl /etc/nginx/

docker-compose-standalone.yml

@@ -632,7 +632,6 @@ services:
       - upload
     ports:
       - "0.0.0.0:443:443"
-      - "0.0.0.0:488:488"
       - "127.0.0.1:5601:5601"
       - "127.0.0.1:9200:9200"
     volumes:

docker-compose.yml

@@ -708,7 +708,6 @@ services:
       - upload
     ports:
       - "0.0.0.0:443:443"
-      - "0.0.0.0:488:488"
       - "127.0.0.1:5601:5601"
       - "127.0.0.1:9200:9200"
     volumes:

docs/authsetup.md

@@ -27,11 +27,11 @@ In either case, you **must** run `./scripts/auth_setup` before starting Malcolm
 
 # <a name="AuthBasicAccountManagement"></a>Local account management
 
-[`auth_setup`](#AuthSetup) is used to define the username and password for the administrator account. Once Malcolm is running, the administrator account can be used to manage other user accounts via a **Malcolm User Management** page served over HTTPS on port 488 (e.g., [https://localhost:488](https://localhost:488) if you are connecting locally).
+[`auth_setup`](#AuthSetup) is used to define the username and password for the administrator account. Once Malcolm is running, the administrator account can be used to manage other user accounts via a **Malcolm User Management** page at [https://localhost/auth](https://localhost/auth/) if you are connecting locally)
 
 Malcolm user accounts can be used to access the [interfaces](quickstart.md#UserInterfaceURLs) of all of its [components](components.md#Components), including Arkime. Arkime uses its own internal database of user accounts, so when a Malcolm user account logs in to Arkime for the first time Malcolm creates a corresponding Arkime user account automatically. This being the case, it is *not* recommended to use the Arkime **Users** settings page or change the password via the **Password** form under the Arkime **Settings** page, as those settings would not be consistently used across Malcolm.
 
-Users may change their passwords via the **Malcolm User Management** page by clicking **User Self Service**. A forgotten password can also be reset via an emailed link, though this requires SMTP server settings to be specified in `htadmin/config.ini` in the Malcolm installation directory.
+Users may change their passwords via the **Malcolm User Management** page by clicking **User Self Service**.
 
 ## <a name="AuthLDAP"></a>Lightweight Directory Access Protocol (LDAP) authentication
 

docs/development.md

@@ -90,7 +90,7 @@ Administrator username: analyst
 analyst password:
 analyst password (again):
 
-Additional local accounts can be created at https://localhost:488/ when Malcolm is running
+Additional local accounts can be created at https://localhost/auth/ when Malcolm is running
 
 (Re)generate self-signed certificates for HTTPS access (Y/n): y 
 
@@ -129,7 +129,7 @@ A minute or so after starting Malcolm, the following services will be accessible
   - PCAP upload (web): https://localhost/upload/
   - PCAP upload (sftp): sftp://[email protected]:8022/files/
   - NetBox: https://localhost/netbox/
-  - Account management: https://localhost:488/  
+  - Account management: https://localhost/auth/
   - Documentation: https://localhost/readme/
 ```
 

docs/opensearch-instances.md

@@ -55,7 +55,7 @@ OpenSearch username: servicedb
 servicedb password:
 servicedb password (again):
 
-Additional local accounts can be created at https://localhost:488/ when Malcolm is running
+Additional local accounts can be created at https://localhost/auth/ when Malcolm is running
 
 Require SSL certificate validation for OpenSearch communication? (Y/n): n
 

docs/quickstart.md

@@ -93,4 +93,4 @@ A few minutes after starting Malcolm (probably 5 to 10 minutes for Logstash to b
 * [Capture File and Log Archive Upload (Web)](upload.md#Upload): [https://localhost/upload/](https://localhost/upload/)
 * [Capture File and Log Archive Upload (SFTP)](upload.md#Upload): `sftp://<username>@127.0.0.1:8022/files`
 * [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis): [https://localhost/netbox/](https://localhost/netbox/)
-* [Account Management](authsetup.md#AuthBasicAccountManagement): [https://localhost:488](https://localhost:488)
+* [Account Management](authsetup.md#AuthBasicAccountManagement): [https://localhost/auth/](https://localhost/auth/)

docs/ubuntu-install-example.md

@@ -224,7 +224,7 @@ Administrator username: analyst
 analyst password:
 analyst password (again):
 
-Additional local accounts can be created at https://localhost:488/ when Malcolm is running
+Additional local accounts can be created at https://localhost/auth/ when Malcolm is running
 
 (Re)generate self-signed certificates for HTTPS access (Y/n): y 
 
@@ -293,7 +293,7 @@ In a few minutes, Malcolm services will be accessible via the following URLs:
   - PCAP upload (web): https://localhost/upload/
   - PCAP upload (sftp): sftp://[email protected]:8022/files/
   - NetBox: https://localhost/netbox/  
-  - Account management: https://localhost:488/
+  - Account management: https://localhost/auth/
   - Documentation: https://localhost/readme/
 
 NAME                           COMMAND                  SERVICE              STATUS               PORTS

htadmin/nginx/sites-available/default

@@ -2,9 +2,10 @@ server {
     listen 80 default_server;
 
     sendfile on;
+    gzip off;
 
     root /var/www/htadmin;
-    index index.php index.html index.htm;
+    index htadmin.php index.html index.htm;
 
     server_name htaccess.malcolm.local;
 

kubernetes/99-nginx-proxy.yml

@@ -11,20 +11,6 @@ spec:
   selector:
     name: nginx-proxy-deployment
 
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx-htadmin-proxy
-  namespace: malcolm
-spec:
-  ports:
-    - port: 488
-      protocol: TCP
-  selector:
-    name: nginx-proxy-deployment
-
-
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -50,7 +36,6 @@ spec:
         ports:
           - containerPort: 443
           - containerPort: 8443
-          - containerPort: 488
         envFrom:
           - configMapRef:
               name: process-env

malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16343117498.desktop

@@ -1,7 +1,7 @@
 [Desktop Entry]
 Version=1.0
 Name=Malcolm - User Management
-Exec=/opt/firefox/firefox https://localhost:488/
+Exec=/opt/firefox/firefox https://localhost/auth/
 Terminal=false
 X-MultipleArgs=false
 Type=Application

malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-users.desktop

@@ -1,6 +1,6 @@
 Version=1.0
 Name=Malcolm - User Management
-Exec=/opt/firefox/firefox https://localhost:488/
+Exec=/opt/firefox/firefox https://localhost/auth/
 Terminal=false
 X-MultipleArgs=false
 Type=Application

nginx/nginx.conf

@@ -80,37 +80,46 @@ http {
     }
   }
 
-  # # htadmin (htpasswd/user management)
-  # server {
-  #   listen 488;
-  #   include /etc/nginx/nginx_ssl_config.conf;
-
-  #   location / {
-  #     proxy_pass http://htadmin;
-  #     proxy_redirect off;
-  #     proxy_set_header Host htadmin.malcolm.local;
-  #   }
-  # }
-
   # Main web interface
   server {
     listen 443;
     include /etc/nginx/nginx_ssl_config.conf;
 
+    # favicon, logos, banners, etc.
+    include /etc/nginx/nginx_image_aliases.conf;
+
+    # HTTP basic user management (doesn't use nginx_auth_rt as it does its own auth directly)
+    location /auth {
+      proxy_pass http://htadmin;
+      proxy_redirect off;
+      proxy_set_header Accept-Encoding "";
+      proxy_set_header Host htadmin.malcolm.local;
+      rewrite ^/auth/?(.*) /$1 break;
+      subs_filter_types '*';
+      subs_filter '(src|action|href)="([\w\.-]+\.(php|css|js))' '$1="/auth/$2' gir;
+      subs_filter 'href="styles/' 'href="/auth/styles/' gi;
+      subs_filter 'src="script/' 'src="/auth/script/' gi;
+      subs_filter '/fonts/glyphicons' '/auth/fonts/glyphicons' gi;
+    }
+    location ~* ^/(htadmin|admin_login)(\.php)\b(.*) {
+      proxy_pass http://htadmin/$1$2$3;
+      proxy_redirect off;
+      proxy_set_header Accept-Encoding "";
+      proxy_set_header Host htadmin.malcolm.local;
+      subs_filter_types '*';
+      subs_filter '(src|action|href)="([\w\.-]+\.(php|css|js))' '$1="/auth/$2' gir;
+      subs_filter 'href="styles/' 'href="/auth/styles/' gi;
+      subs_filter 'src="script/' 'src="/auth/script/' gi;
+      subs_filter '/fonts/glyphicons' '/auth/fonts/glyphicons' gi;
+    }
+
     # Malcolm readme
     location /readme {
       include /etc/nginx/nginx_auth_rt.conf;
       root /usr/share/nginx/html;
       try_files $uri $uri/index.html;
     }
 
-    location /htadmin {
-      rewrite ^/htadmin(.*)/?$ /$1 break;
-      proxy_pass http://htadmin;
-      proxy_redirect off;
-      proxy_set_header Host htadmin.malcolm.local;
-    }
-
     # Malcolm file upload
     location /upload {
       include /etc/nginx/nginx_auth_rt.conf;
@@ -144,8 +153,6 @@ http {
       proxy_set_header Host arkime.malcolm.local;
     }
 
-
-
     # Arkime -> Dashboards shortcut
     location ~* ^/idark2dash(.*) {
       include /etc/nginx/nginx_auth_rt.conf;
@@ -229,9 +236,6 @@ http {
       proxy_set_header X-Remote-Auth $authenticated_user;
     }
 
-    # favicon, logos, banners, etc.
-    include /etc/nginx/nginx_image_aliases.conf;
-
     # Fix cyberchef JS module(s)
     # https://localhost/arkime/session/190924-KgO9H30qhdREw7ltsDXn1Rgp/modules/Regex.js
     location ~* ^/arkime/session/.*/(modules/.*\.js) {

scripts/build.sh

@@ -96,7 +96,7 @@ FILES_IN_IMAGES=(
   "/usr/share/filebeat/filebeat.yml;filebeat-oss"
   "/var/www/upload/js/jquery.fileupload.js;file-upload"
   "/opt/freq_server/freq_server.py;freq"
-  "/var/www/htadmin/index.php;htadmin"
+  "/var/www/htadmin/htadmin.php;htadmin"
   "/etc/ip_protocol_name_to_number.yaml;logstash"
   "/etc/ja3.yaml;logstash"
   "/etc/vendor_macs.yaml;logstash"

scripts/control.py

@@ -702,7 +702,7 @@ def logs():
             print("  - PCAP upload (web): https://localhost/upload/")
             print("  - PCAP upload (sftp): sftp://[email protected]:8022/files/")
             print("  - NetBox: https://localhost/netbox/\n")
-            print("  - Account management: https://localhost:488/\n")
+            print("  - Account management: https://localhost/auth/\n")
             print("  - Documentation: https://localhost/readme/\n")
 
     process.poll()
@@ -1123,25 +1123,25 @@ def authSetup(wipe=False):
                     f.write('; Change this to customize your title:\n')
                     f.write('app_title = Malcolm User Management\n\n')
                     f.write('; htpasswd file\n')
-                    f.write('secure_path  = ./config/htpasswd\n')
+                    f.write('secure_path  = ./config/auth/htpasswd\n')
                     f.write('; metadata file\n')
                     f.write('metadata_path  = ./config/metadata\n\n')
                     f.write('; administrator user/password (htpasswd -b -c -B ...)\n')
                     f.write(f'admin_user = {username}\n\n')
                     f.write('; username field quality checks\n')
                     f.write(';\n')
                     f.write('min_username_len = 4\n')
-                    f.write('max_username_len = 12\n\n')
+                    f.write('max_username_len = 32\n\n')
                     f.write('; Password field quality checks\n')
                     f.write(';\n')
-                    f.write('min_password_len = 6\n')
-                    f.write('max_password_len = 20\n\n')
+                    f.write('min_password_len = 8\n')
+                    f.write('max_password_len = 128\n\n')
 
                 # touch the metadata file
                 open(os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), 'a').close()
 
                 DisplayMessage(
-                    'Additional local accounts can be created at https://localhost:488/ when Malcolm is running',
+                    'Additional local accounts can be created at https://localhost/auth/ when Malcolm is running',
                 )
 
             # generate HTTPS self-signed certificates

scripts/malcolm_appliance_packager.sh

@@ -157,7 +157,7 @@ if mkdir "$DESTDIR"; then
   echo "  - PCAP upload (web): https://localhost/upload/" | tee -a "$README"
   echo "  - PCAP upload (sftp): sftp://[email protected]:8022/files/" | tee -a "$README"
   echo "  - NetBox: https://localhost/netbox/" | tee -a "$README"
-  echo "  - Account management: https://localhost:488/" | tee -a "$README"
+  echo "  - Account management: https://localhost/auth/" | tee -a "$README"
   echo "  - Documentation: https://localhost/readme/" | tee -a "$README"
   popd  >/dev/null 2>&1
   popd  >/dev/null 2>&1