Merge pull request #1049 from hashtopolis/dev

merge updates from dev to master for next release
hashtopolis · Mar 19, 2024 · fb352c2 · fb352c2
2 parents 623136c + ddb3cc9
commit fb352c2
Show file tree

Hide file tree

Showing 46 changed files with 1,158 additions and 227 deletions.
diff --git a/ci/apiv2/HACKING.md b/ci/apiv2/HACKING.md
@@ -14,7 +14,13 @@ curl --header "Content-Type: application/json" -X GET --header "Authorization: B
 
 Access database:
 ```
-mysql -u $HASHTOPOLIS_DB_USER -p $HASHTOPOLIS_DB_PASS -h $HASHTOPOLIS_DB_HOST -D $HASHTOPOLIS_DB_DATABASE
+mysql -u $HASHTOPOLIS_DB_USER -p$HASHTOPOLIS_DB_PASS -h $HASHTOPOLIS_DB_HOST -D $HASHTOPOLIS_DB_DATABASE
+```
+
+Enable query logging:
+```
+docker exec $(docker ps -aqf "ancestor=mysql:8.0") mysql -u root -phashtopolis -e "SET global log_output = 'FILE'; SET global general_log_file='/tmp/mysql_all.log'; SET global general_log = 1;"
+docker exec $(docker ps -aqf "ancestor=mysql:8.0") tail -f /tmp/mysql_all.log
 ```
 
 ### paper flipchart scribbles

diff --git a/ci/apiv2/generate_dummy_data.py b/ci/apiv2/generate_dummy_data.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+"""
+Generate dummy data for development/debugging purposes
+"""
+from utils import do_create_hashlist
+
+
+def generate_dummy_data():
+    for _ in range(1000):
+        # TODO: Generate unique hashlists
+        do_create_hashlist()
+
+
+# TODO: Generate different objects like users/tasks/crackerbinaries/etc
+if __name__ == '__main__':
+    # TODO: Use seed to generate an predictable 'random' test dataset
+    generate_dummy_data()
diff --git a/ci/apiv2/hashtopolis.py b/ci/apiv2/hashtopolis.py
@@ -683,3 +683,56 @@ def purge_task(self, task):
             'taskId': task.id,
         }
         return self._helper_request("purgeTask", payload)
+
+    def export_cracked_hashes(self, hashlist):
+        payload = {
+            'hashlistId': hashlist.id,
+        }
+        response = self._helper_request("exportCrackedHashes", payload)
+        return File(**response['data'])
+
+    def export_left_hashes(self, hashlist):
+        payload = {
+            'hashlistId': hashlist.id,
+        }
+        response = self._helper_request("exportLeftHashes", payload)
+        return File(**response['data'])
+
+    def export_wordlist(self, hashlist):
+        payload = {
+            'hashlistId': hashlist.id,
+        }
+        response = self._helper_request("exportWordlist", payload)
+        return File(**response['data'])
+
+    def import_cracked_hashes(self, hashlist, source_data, separator):
+        payload = {
+            'hashlistId': hashlist.id,
+            'sourceData': source_data,
+            'separator': separator,
+        }
+        response = self._helper_request("importCrackedHashes", payload)
+        return response['data']
+
+
+    def recount_file_lines(self, file):
+        payload = {
+            'fileId': file.id,
+        }
+        response = self._helper_request("recountFileLines", payload)
+        return File(**response['data'])
+
+    def unassign_agent(self, agent):
+        payload = {
+            'agentId': agent.id,
+        }
+        response = self._helper_request("unassignAgent", payload)
+        return response['data']
+
+    def assign_agent(self, agent, task):
+        payload = {
+            'agentId': agent.id,
+            'taskId': task.id,
+        }
+        response = self._helper_request("assignAgent", payload)
+        return response['data']
diff --git a/ci/apiv2/test_agent.py b/ci/apiv2/test_agent.py
@@ -1,4 +1,5 @@
-from hashtopolis import Agent
+from test_task import TaskTest
+from hashtopolis import Agent, Helper
 from hashtopolis import HashtopolisError
 
 from utils import BaseTest
@@ -28,3 +29,21 @@ def test_expandables(self):
         model_obj = self.create_test_object()
         expandables = ['accessGroups', 'agentstats']
         self._test_expandables(model_obj, expandables)
+
+    def test_assign_unassign_agent(self):
+        agent_obj = self.create_test_object()
+
+        task_test = TaskTest()
+        task_obj = task_test.create_test_object(delete=True)
+
+        helper = Helper()
+
+        result = helper.assign_agent(agent=agent_obj, task=task_obj)
+
+        self.assertEqual(result['assign'], 'success')
+
+        result = helper.unassign_agent(agent=agent_obj)
+
+        self.assertEqual(result['unassign'], 'success')
+
+        task_test.tearDown()
diff --git a/ci/apiv2/test_file.py b/ci/apiv2/test_file.py
@@ -1,4 +1,4 @@
-from hashtopolis import File
+from hashtopolis import File, Helper
 from utils import BaseTest
 
 
@@ -28,3 +28,14 @@ def test_expandables(self):
     def test_create_binary(self):
         model_obj = self.create_test_object(compress=True)
         self._test_create(model_obj)
+
+    def test_recount_wordlist(self):
+        # Note: After the object creation, the line count is already updated, but afterward it is immutable on the API.
+        #       There the test just check that the API function is callable and returns the file, but the count is
+        #       already the same beforehand.
+        model_obj = self.create_test_object()
+
+        helper = Helper()
+        file = helper.recount_file_lines(file=model_obj)
+
+        self.assertEqual(file.lineCount, 3)
diff --git a/ci/apiv2/test_filter_and_ordering.py b/ci/apiv2/test_filter_and_ordering.py
@@ -24,7 +24,6 @@ def test_filter(self):
             [x.id for x in model_objs],
             [x.id for x in objs])
 
-    @pytest.mark.skip(reason="Broken due to bug https://github.com/hashtopolis/server/issues/968")
     def test_filter__contains(self):
         search_token = "SHA"
         objs = HashType.objects.filter(description__contains=search_token)
@@ -33,7 +32,6 @@ def test_filter__contains(self):
             [x.id for x in all_objs if search_token in x.description],
             [x.id for x in objs])
 
-    @pytest.mark.skip(reason="Broken due to bug https://github.com/hashtopolis/server/issues/968")
     def test_filter__endswith(self):
         search_token = 'sha512'
         objs = HashType.objects.filter(description__endswith=search_token)
@@ -108,7 +106,6 @@ def test_filter__ne(self):
             [x.id for x in all_objs if x.hashTypeId != 100],
             [x.id for x in objs])
 
-    @pytest.mark.skip(reason="Broken due to bug https://github.com/hashtopolis/server/issues/968")
     def test_filter__startswith(self):
         objs = HashType.objects.filter(description__startswith="net")
         all_objs = HashType.objects.all()

diff --git a/ci/apiv2/test_hashlist.py b/ci/apiv2/test_hashlist.py
@@ -1,4 +1,4 @@
-from hashtopolis import Hashlist, Helper
+from hashtopolis import Hashlist, Helper, File
 from utils import BaseTest
 
 
@@ -38,6 +38,104 @@ def test_create_alternative_hashtype(self):
         model_obj = self.create_test_object(file_id='003')
         self._test_create(model_obj)
 
+    def test_export_cracked_hashes(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        helper = Helper()
+        file = helper.export_cracked_hashes(model_obj)
+
+        obj = File.objects.get(fileId=file.id)
+        self.assertEqual(int(file.id), obj.id)
+        self.assertIn('Pre-cracked_', obj.filename)
+
+        self.delete_after_test(obj)
+
+    def test_export_left_hashes(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        helper = Helper()
+        file = helper.export_left_hashes(model_obj)
+
+        obj = File.objects.get(fileId=file.id)
+        self.assertEqual(int(file.id), obj.id)
+        self.assertIn('Leftlist_', obj.filename)
+
+        self.delete_after_test(obj)
+
+    def test_export_wordlist(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
+
+        helper = Helper()
+        helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        file = helper.export_wordlist(model_obj)
+
+        obj = File.objects.get(fileId=file.id)
+        self.assertEqual(int(file.id), obj.id)
+        self.assertIn('Wordlist_', obj.filename)
+
+        self.delete_after_test(obj)
+
+    def test_import_cracked_hashes(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
+
+        helper = Helper()
+        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        self.assertEqual(result['totalLines'], 1)
+        self.assertEqual(result['newCracked'], 1)
+
+        obj = Hashlist.objects.get(hashlistId=model_obj.id)
+        self.assertEqual(obj.cracked, 1)
+
+    def test_import_cracked_hashes_invalid(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        cracked = "cc03e747a6afbbcbf8be7668acfebee5__test123"
+
+        helper = Helper()
+        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        self.assertEqual(result['totalLines'], 1)
+        self.assertEqual(result['invalid'], 1)
+
+        obj = Hashlist.objects.get(hashlistId=model_obj.id)
+        self.assertEqual(obj.cracked, 0)
+
+    def test_import_cracked_hashes_notfound(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        cracked = "ffffffffffffffffffffffffffffffff:test123"
+
+        helper = Helper()
+        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        self.assertEqual(result['totalLines'], 1)
+        self.assertEqual(result['notFound'], 1)
+
+        obj = Hashlist.objects.get(hashlistId=model_obj.id)
+        self.assertEqual(obj.cracked, 0)
+
+    def test_import_cracked_hashes_already_cracked(self):
+        model_obj = self.create_test_object(file_id='001')
+
+        cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
+
+        helper = Helper()
+        helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+
+        self.assertEqual(result['totalLines'], 1)
+        self.assertEqual(result['alreadyCracked'], 1)
+
+        obj = Hashlist.objects.get(hashlistId=model_obj.id)
+        self.assertEqual(obj.cracked, 1)
+
     def test_helper_create_superhashlist(self):
         hashlists = [self.create_test_object() for _ in range(2)]
 

diff --git a/ci/apiv2/test_task.py b/ci/apiv2/test_task.py
@@ -1,4 +1,4 @@
-from hashtopolis import Task
+from hashtopolis import Task, TaskWrapper
 from utils import BaseTest
 
 
@@ -75,3 +75,27 @@ def test_task_with_file(self):
         task = self.create_task(hashlist, extra_payload=extra_payload)
         obj = Task.objects.get(pk=task.id, expand='files')
         self.assertListEqual([x.id for x in files], [x.id for x in obj.files_set])
+
+    def test_task_update_priority(self):
+        task = self.create_test_object()
+        obj = TaskWrapper.objects.get(pk=task.taskWrapperId)
+        self.assertEqual(task.priority, obj.priority)
+
+        new_priority = task.priority + 1234
+        task.priority = new_priority
+        task.save()
+
+        obj = TaskWrapper.objects.get(pk=task.taskWrapperId)
+        self.assertEqual(new_priority, obj.priority)
+
+    def test_task_update_maxagent(self):
+        task = self.create_test_object()
+        obj = TaskWrapper.objects.get(pk=task.taskWrapperId)
+        self.assertEqual(task.maxAgents, obj.maxAgents)
+
+        new_maxagent = task.maxAgents + 1234
+        task.maxAgents = new_maxagent
+        task.save()
+
+        obj = TaskWrapper.objects.get(pk=task.taskWrapperId)
+        self.assertEqual(new_maxagent, obj.maxAgents)
diff --git a/src/api/v2/index.php b/src/api/v2/index.php
@@ -267,11 +267,18 @@ public function process(Request $request, RequestHandler $handler): Response {
 require __DIR__ . "/../../inc/apiv2/model/vouchers.routes.php";
 
 require __DIR__ . "/../../inc/apiv2/helper/abortChunk.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/assignAgent.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/createSupertask.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/createSuperHashlist.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/exportCrackedHashes.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/exportLeftHashes.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/exportWordlist.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/importCrackedHashes.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/importFile.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/purgeTask.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/recountFileLines.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/resetChunk.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/setUserPassword.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/unassignAgent.routes.php";
 
 $app->run();
diff --git a/src/bin/hashtopolis.zip b/src/bin/hashtopolis.zip
diff --git a/src/dba/AbstractModelFactory.class.php b/src/dba/AbstractModelFactory.class.php
@@ -114,7 +114,7 @@ public function save($model) {
     $stmt = $dbh->prepare($query);
     $stmt->execute($vals);
 
-    $id = $dbh->lastInsertId();
+    $id = intval($dbh->lastInsertId());
     if ($id != 0) {
       $model->setId($id);
       return $model;