[pull] master from danielmiessler:master

.bin/file-extensions-downloader.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+
+# Open a pull req if any sources are missing/need to be added
+
+from bs4 import BeautifulSoup
+import requests
+import time
+import re
+
+MICROSOFT_URL="https://support.microsoft.com/en-us/windows/common-file-name-extensions-in-windows-da4a4430-8e76-89c5-59f7-1cdbbc75cb01"
+WIKI_ROOT="https://en.wikipedia.org"
+WIKI_URL="https://en.wikipedia.org/wiki/List_of_filename_extensions"
+GFG_URL="https://www.geeksforgeeks.org/list-of-file-formats/"
+
+wiki_urls=[]
+exts=[]
+tables=[]
+
+res=requests.get(WIKI_URL).text
+wiki_soup=BeautifulSoup(res,"html.parser")
+
+for i in wiki_soup.findAll('div',{"class":"mw-content-ltr mw-parser-output"})[0].findAll("ul")[2].findAll("li"):
+    wiki_urls.append(WIKI_ROOT+i.a.get('href'))
+
+for i in wiki_urls:
+    res=requests.get(i).text
+    wiki_soup=BeautifulSoup(res,"html.parser")
+
+    tables+=wiki_soup.findAll("table")
+    time.sleep(0.5)
+
+for i in tables:
+
+    if "wikitable" not in i["class"]:
+        continue
+
+
+    for ext in i.tbody.findAll("tr"):
+        ext=ext.findAll('td')
+
+        if ext==[]:
+            continue
+
+        ext=re.sub(r"\[.*?\]","",ext[0].text).strip()
+
+        if "," in ext:
+            for j in ext.split(","):
+
+                if "." in j:
+                    continue
+
+                exts.append(j.strip())
+            continue
+
+        if "." in ext:
+            continue
+
+        exts.append(ext)
+
+res=requests.get(MICROSOFT_URL).text
+microsoft_soup=BeautifulSoup(res,"html.parser")
+
+microsoft_exts=microsoft_soup.findAll("tbody")[1].findAll('p')[::2]
+
+for i in microsoft_exts:
+
+    i=i.text
+
+    if "," in i:
+        i=i.split(",")
+        for j in i:
+            exts.append(j.strip())
+        continue
+
+    exts.append(i)
+
+res=requests.get(GFG_URL).text
+gfg_soup=BeautifulSoup(res,"html.parser")
+
+gfg_exts=gfg_soup.findAll("tbody")
+
+for i in gfg_exts:
+    i=i.findAll('th')
+    for ext in i:
+        ext=ext.text.strip()
+        if ext.startswith('.'):
+            exts.append(ext[1:].upper())
+        else:
+            exts.append(ext.upper())
+
+cleaned_exts=[]
+for i in exts:
+
+    # https://stackoverflow.com/questions/3627784/case-insensitive-in
+    if i.upper() in (cleaned_ext.upper() for cleaned_ext in cleaned_exts):
+        continue
+
+    cleaned_exts.append(i)
+
+exts=cleaned_exts
+
+exts=list(dict.fromkeys(exts))
+exts.sort()
+
+open("../Fuzzing/file-extensions.txt","w").write("\n".join(exts))
+
+mutated_exts=[]
+
+for i in exts:
+    mutated_exts.append(i)
+    mutated_exts.append(i.upper())
+    mutated_exts.append(i.lower())
+
+mutated_exts=list(dict.fromkeys(mutated_exts))
+mutated_exts.sort()
+
+open("../Fuzzing/file-extensions-all-cases.txt","w").write("\n".join(mutated_exts))
+
+mutated_exts=[]
+
+for i in exts:
+    mutated_exts.append(i.lower())
+
+mutated_exts=list(dict.fromkeys(mutated_exts))
+mutated_exts.sort()
+
+open("../Fuzzing/file-extensions-lower-case.txt","w").write("\n".join(mutated_exts))
+
+mutated_exts=[]
+
+for i in exts:
+    mutated_exts.append(i.upper())
+
+mutated_exts=list(dict.fromkeys(mutated_exts))
+mutated_exts.sort()
+
+open("../Fuzzing/file-extensions-upper-case.txt","w").write("\n".join(mutated_exts))

.bin/new-line-and-empty-line-checker.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt"
+
+import os
+import sys
+
+print("[+] New line check")
+
+if not sys.argv[1]:
+    exit(0)
+
+files=sys.argv[1].split(" ")
+
+for i in files:
+    if not os.path.isfile(i):
+        print("[!] %s does not exist!"%(i))
+        exit(2)
+
+for i in files:
+    contents=open(i,"rb").read()
+
+    if contents[-1] == b'\n':
+        print("[!] %s ends with a new line!"%(i))
+        exit(2)
+    print("[+] %s passed new line check!"%(i))
+
+    counter=1
+
+    for line in contents.split(b'\n'):
+        if len(line)==0:
+            print("[!] %s has an empty entry at line %i!"%(i,counter))
+            exit(2)
+        counter+=1
+    print("[+] %s passed empty line check!"%(i))
+
+print("[+] All files passed checks")
+# exit(0)

.bin/os-names-mutate.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+text=open('../Fuzzing/os-names.txt').read().split('\n')
+new_temp=[]
+for i in text:
+ if " " in i:
+  new_temp.append(i.replace(" ", "-"))
+  new_temp.append(i.replace(" ", "_"))
+ else:
+  new_temp.append(i)
+temp=[]
+for i in new_temp:
+ if i.lower() != i:
+  temp.append(i)
+  temp.append(i.lower())
+ else:
+  temp.append(i)
+
+open("../Fuzzing/os-names-mutated.txt","w").write('\n'.join(temp))

.bin/trickest-patcher.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+import os,shutil
+
+print("[+] trickest wordlist patcher")
+
+ROOT=".working_space"
+INPUT_TECHNOLOGIES=os.path.join(ROOT,"wordlists/technologies/")
+INPUT_ROBOTS=os.path.join(ROOT,"wordlists/robots/")
+OUTPUT_TECHNOLOGIES="Discovery/Web-Content/CMS/trickest-cms-wordlist/"
+OUTPUT_ROBOTS="Discovery/Web-Content/trickest-robots-disallowed-wordlists/"
+
+if not os.path.isdir(".working_space"):
+    print("[!] Working dir not found!")
+    exit(2)
+
+if not os.path.isdir(os.path.join(ROOT,"wordlists")):
+    print("[!] wordlists dir not found!")
+    exit(2)
+
+if not os.path.isdir(OUTPUT_TECHNOLOGIES):
+    os.makedirs(OUTPUT_TECHNOLOGIES)
+
+if not os.path.isdir(OUTPUT_ROBOTS):
+    os.makedirs(OUTPUT_ROBOTS)
+
+for i in os.listdir(INPUT_TECHNOLOGIES):
+    path=os.path.join(INPUT_TECHNOLOGIES,i)
+
+    if os.path.isfile(path):
+        shutil.copy(path,OUTPUT_TECHNOLOGIES)
+    else:
+        shutil.copytree(path,OUTPUT_TECHNOLOGIES,dirs_exist_ok=True)
+
+for i in os.listdir(INPUT_ROBOTS):
+    path=os.path.join(INPUT_ROBOTS,i)
+
+    if os.path.isfile(path):
+        shutil.copy(path,OUTPUT_ROBOTS)
+    else:
+        shutil.copytree(path,OUTPUT_ROBOTS,dirs_exist_ok=True)
+
+print("[+] Copied all the files")
+for i in [OUTPUT_ROBOTS,OUTPUT_TECHNOLOGIES]:
+    for root,_,file_list in os.walk(i):
+        for file in file_list:
+
+            path=os.path.join(root,file)
+            contents=open(path,"rb").read()
+
+            if contents.endswith(b"\n"):
+                print("[!] %s ends with new line"%(path))
+                contents=contents[:-1]
+                open(path,"wb").write(contents)
+
+            patch_content=[]
+            counter=0
+            for content in contents.split(b"\n"):
+                counter+=1
+                if not content:
+                    print("[+] %s has an empty line at %i"%(path,counter))
+                    continue
+                patch_content.append(content)
+
+            if len(contents)!=len(patch_content):
+                open(path,"wb").write(b"\n".join(patch_content))
+
+
+

.bin/trickest-updater.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/bash
+
+set -e 
+set -o pipefail
+
+mkdir -p .working_space
+cd .working_space
+git clone --depth=1 https://github.com/trickest/wordlists.git
+cd ../
+
+./.bin/trickest-patcher.py
+rm -rf .working_space

.bin/validators.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# https://stackoverflow.com/questions/3822621/how-to-exit-if-a-command-failed
+
+set -e 
+set -o pipefail
+
+# wrapper for all the checking scripts
+echo $1
+./.bin/check-file-for-starting-slash "$1"
+./.bin/new-line-and-empty-line-checker.py "$1"

.github/workflows/wordlist-updater_trickest-wordlists.yml

@@ -0,0 +1,32 @@
+name: Wordlist Updater - Trickest wordlists updater
+
+on:
+  schedule:
+  - cron: 0 0 * * *
+
+  workflow_dispatch:
+
+jobs:
+  update-files:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Update lists
+        run: ./.bin/trickest-updater.sh
+
+      - name: Commit files if changed
+        run: |
+          git add -N .
+
+          if [ -z "$(git ls-files --modified)" ]; then
+              echo "[+] No files were changed"
+          else
+              echo "[+] Files were changed! Pushing changed..."
+              git add -A
+              git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
+              git config --local user.email "[email protected]"
+              git config --local user.name "GitHub Action"
+              git commit -m "[Github Action] Automated trickest wordlists update."
+              git push
+          fi

.github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml → .github/workflows/wordlist-validator.yml

@@ -2,8 +2,10 @@
 # Sources:
 # https://dev.to/scienta/get-changed-files-in-github-actions-1p36
 # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions
-# https://github.com/marketplace/actions/changed-files
-name: Wordlist Validator - Verify if any file entry start with a slash
+# https://github.com/marketplace/actions/changed-files\
+#
+# Modified to add a validator script
+name: Wordlist Validator - Runs a validator script to check for dangerous pushes
 on:
   push:
     paths:
@@ -26,5 +28,4 @@ jobs:
         uses: tj-actions/changed-files@v34
       - name: Analyze all added or modified files
         run: |
-          chmod +x ./.bin/check-file-for-starting-slash
-          ./.bin/check-file-for-starting-slash "${{ steps.changed-files.outputs.all_changed_files }}"
+          ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}"

.gitignore

@@ -2,3 +2,4 @@
 .*.icloud
 .gitkeep
 .idea
+.working_space/