Merge pull request #1428 from Yamato-Security/add-integration-test-fo…

…r-encoded-rules test: add integration-test/csv(json)-timeline diff-test for encoded_rules/single config
Yamato-Security · Oct 8, 2024 · 5dad150 · 5dad150
2 parents 1b94b51 + 92bb6d9
commit 5dad150
Show file tree

Hide file tree

Showing 2 changed files with 145 additions and 7 deletions.
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -42,22 +42,22 @@ jobs:
         run: cd main && cargo run --release -- computer-metrics -d ../hayabusa-sample-evtx -q
 
       - name: csv-timeline
-        run: cd main && cargo run --release -- csv-timeline -d ../hayabusa-sample-evtx -o out.csv -q -w
+        run: cd main && cargo run --release -- csv-timeline -d ../hayabusa-sample-evtx -o out.csv -q -w -D -n -u
 
       - name: csv-timeline(-p super-verbose)
-        run: cd main && cargo run --release -- csv-timeline -d ../hayabusa-sample-evtx -o out-s.csv -p super-verbose -q -w
+        run: cd main && cargo run --release -- csv-timeline -d ../hayabusa-sample-evtx -o out-s.csv -p super-verbose -q -w -D -n -u
 
       - name: eid-metrics
         run: cd main && cargo run --release -- eid-metrics -d ../hayabusa-sample-evtx -q
 
       - name: json-timeline
-        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out.json -q -w
+        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out.json -q -w -D -n -u
 
       - name: json-timeline(-p super-verbose)
-        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out-s.json -p super-verbose -q -w
+        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out-s.json -p super-verbose -q -w -D -n -u
 
       - name: json-timeline(-L)
-        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out.jsonl -q -L -w
+        run: cd main && cargo run --release -- json-timeline -d ../hayabusa-sample-evtx -o out.jsonl -q -L -w -D -n -u
 
       - name: level-tuning
         run: cd main && cargo run --release -- level-tuning -f ./rules/config/level_tuning.txt -q
@@ -78,4 +78,98 @@ jobs:
         run: cd main && cargo run --release -- search -d ../hayabusa-sample-evtx -k mimikatz -o search.csv -q
 
       - name: set-default-profile
-        run: cd main && cargo run --release -- set-default-profile -p verbose -q
+        run: cd main && cargo run --release -- set-default-profile -p verbose -q
+
+  integration-test-encoded-rules:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: ['ubuntu-24.04', 'windows-2022']
+
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          path: main
+
+      - name: Checkout hayabusa-sample-evtx repo
+        uses: actions/checkout@v4
+        with:
+          repository: Yamato-Security/hayabusa-sample-evtx
+          path: hayabusa-sample-evtx
+
+      - name: Set up Rust toolchain
+        if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Download encoded rules and config files (Windows)
+        if: contains(matrix.os, 'windows') == true
+        shell: pwsh
+        run: |
+          cd main 
+          cargo build --release
+          Copy-Item -Path target\release\hayabusa.exe -Destination .\hayabusa.exe
+          Remove-Item -Path ./config -Recurse -Force
+          Remove-Item -Path ./rules -Recurse -Force
+          Invoke-WebRequest -Uri https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/encoded_rules.yml -OutFile encoded_rules.yml
+          Invoke-WebRequest -Uri https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt -OutFile rules_config_files.txt
+
+      - name: Download encoded rules and config files (Ubuntu)
+        if: contains(matrix.os, 'ubuntu') == true
+        run: |
+          cd main
+          cargo build --release
+          cp target/release/hayabusa .
+          rm -rf ./config && rm -rf ./rules/
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/encoded_rules.yml
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt
+
+      - name: help
+        run: cd main && ./hayabusa help
+
+      - name: update-rules
+        run: cd main && ./hayabusa update-rules -q
+
+      - name: computer-metrics
+        run: cd main && ./hayabusa computer-metrics -d ../hayabusa-sample-evtx -q
+
+      - name: csv-timeline
+        run: cd main && ./hayabusa csv-timeline -d ../hayabusa-sample-evtx -o out.csv -q -w -D -n -u
+
+      - name: csv-timeline(-p super-verbose)
+        run: cd main && ./hayabusa csv-timeline -d ../hayabusa-sample-evtx -o out-s.csv -p super-verbose -q -w -D -n -u
+
+      - name: eid-metrics
+        run: cd main && ./hayabusa eid-metrics -d ../hayabusa-sample-evtx -q
+
+      - name: json-timeline
+        run: cd main && ./hayabusa json-timeline -d ../hayabusa-sample-evtx -o out.json -q -w -D -n -u
+
+      - name: json-timeline(-p super-verbose)
+        run: cd main && ./hayabusa json-timeline -d ../hayabusa-sample-evtx -o out-s.json -p super-verbose -q -w -D -n -u
+
+      - name: json-timeline(-L)
+        run: cd main && ./hayabusa json-timeline -d ../hayabusa-sample-evtx -o out.jsonl -q -L -w -D -n -u
+
+      - name: level-tuning
+        run: cd main && ./hayabusa level-tuning -f ./rules/config/level_tuning.txt -q
+
+      - name: list-contributors
+        run: cd main && ./hayabusa list-contributors -q
+
+      - name: list-profiles
+        run: cd main && ./hayabusa list-profiles -q
+
+      - name: logon-summary
+        run: cd main && ./hayabusa logon-summary -d ../hayabusa-sample-evtx -q
+
+      - name: pivot-keywords-list
+        run: cd main && ./hayabusa pivot-keywords-list -d ../hayabusa-sample-evtx -o key -q -w
+
+      - name: search
+        run: cd main && ./hayabusa search -d ../hayabusa-sample-evtx -k mimikatz -o search.csv -q
+
+      - name: set-default-profile
+        run: cd main && ./hayabusa set-default-profile -p verbose -q
diff --git a/.github/workflows/timeline-diff.yml b/.github/workflows/timeline-diff.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  timeline-diff:
     runs-on: ubuntu-latest
     env:
       BRANCH_NAME: ${{ github.head_ref }}
@@ -32,13 +32,41 @@ jobs:
           cargo run --release -- csv-timeline -d ./hayabusa-sample-evtx -o dev.csv -p super-verbose -q -w -D -n -u
           cargo run --release -- json-timeline -d ./hayabusa-sample-evtx -o dev.jsonl -L -p super-verbose -q -w -D -n -u
 
+      - name: Run on dev branch(encoded_rules)
+        run: |
+          cargo build --release
+          mv ./config ../
+          mv ./rules ../
+          cp target/release/hayabusa .
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/encoded_rules.yml
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt
+          ./hayabusa update-rules -q
+          ./hayabusa csv-timeline -d ./hayabusa-sample-evtx -o dev-encoded.csv -p super-verbose -q -w -D -n -u
+          ./hayabusa json-timeline -d ./hayabusa-sample-evtx -o dev-encoded.jsonl -L -p super-verbose -q -w -D -n -u
+          mv ../config ./
+          mv ../rules ./
+          mv encoded_rules.yml ../
+          mv rules_config_files.txt ../
+
       - name: Run on main branch
         run: |
           git checkout main
           cargo run --release -- update-rules -q
           cargo run --release -- csv-timeline -d ./hayabusa-sample-evtx -o main.csv -p super-verbose -q -w -D -n -u
           cargo run --release -- json-timeline -d ./hayabusa-sample-evtx -o main.jsonl -L -p super-verbose -q -w -D -n -u
 
+      - name: Run on main branch(encoded_rules)
+        run: |
+          cargo build --release
+          mv ./config ../
+          mv ./rules ../
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/encoded_rules.yml
+          curl -O https://raw.githubusercontent.com/Yamato-Security/hayabusa-encoded-rules/refs/heads/main/rules_config_files.txt
+          cp target/release/hayabusa .
+          ./hayabusa update-rules -q
+          ./hayabusa csv-timeline -d ./hayabusa-sample-evtx -o main-encoded.csv -p super-verbose -q -w -D -n -u
+          ./hayabusa json-timeline -d ./hayabusa-sample-evtx -o main-encoded.jsonl -L -p super-verbose -q -w -D -n -u
+          rm -rf encoded_rules.yml rules_config_files.txt hayabusa
 
       - name: Check CSV Timeline diff
         run: |
@@ -48,7 +76,23 @@ jobs:
             exit 1
           fi
 
+      - name: Check CSV Timeline diff(encoded_rules)
+        run: |
+          diff main-encoded.csv dev-encoded.csv
+          if [ $? -ne 0 ]; then
+            echo "CSV files are different"
+            exit 1
+          fi
+
       - name: Check JSONL Timeline diff
+        run: |
+          diff main.jsonl dev.jsonl
+          if [ $? -ne 0 ]; then
+            echo "JSON files are different"
+            exit 1
+          fi
+
+      - name: Check JSONL Timeline diff(encoded_rules)
         run: |
           diff main.jsonl dev.jsonl
           if [ $? -ne 0 ]; then