diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ec54468e..f44f73db6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.12.0] - 2024-12-14 + +- Reduce the check-in tests that were adding an unncessary amount of time to the CI +- Allow Python 3.13 to be used +- Fix python install - Support Empire for system-wide deployment (@D3vil0p3r) - Paths specified in config.yaml where user does not have write permission will be fallback to ~/.empire directory and config.yaml updated as well (@D3vil0p3r) - Invoke-Obfuscation is no longer copied to /usr/local/share @@ -948,7 +953,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated shellcoderdi to newest version (@Cx01N) - Added a Nim launcher (@Hubbl3) -[Unreleased]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.7...HEAD +[Unreleased]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.12.0...HEAD + +[5.12.0]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.7...v5.12.0 [5.11.7]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.6...v5.11.7 diff --git a/empire/server/common/empire.py b/empire/server/common/empire.py index c4495f2c0..f77cfafaa 100755 --- a/empire/server/common/empire.py +++ b/empire/server/common/empire.py @@ -38,7 +38,7 @@ from . import agents, credentials, listeners, stagers -VERSION = "5.11.7 BC Security Fork" +VERSION = "5.12.0 BC Security Fork" log = logging.getLogger(__name__) diff --git a/empire/test/conftest.py b/empire/test/conftest.py index 5b034935c..6c96d9461 100644 --- a/empire/test/conftest.py +++ b/empire/test/conftest.py @@ -1,4 +1,3 @@ -import asyncio import os import shutil import sys @@ -57,7 +56,8 @@ def client(): # fix for pycharm debugger # https://stackoverflow.com/a/77926544/5849681 - yield TestClient(app, backend_options={"loop_factory": asyncio.new_event_loop}) + # yield TestClient(app, backend_options={"loop_factory": asyncio.new_event_loop}) + yield TestClient(app) from empire.server.server import main diff --git a/empire/test/test_agent_checkins_api.py b/empire/test/test_agent_checkins_api.py index 19ae2c595..5fa044405 100644 --- a/empire/test/test_agent_checkins_api.py +++ b/empire/test/test_agent_checkins_api.py @@ -1,7 +1,5 @@ import asyncio import logging -import time -from contextlib import contextmanager from datetime import datetime, timedelta, timezone import pytest @@ -10,12 +8,6 @@ log = logging.getLogger(__name__) -@contextmanager -def timer(): - start = time.perf_counter() - yield lambda: time.perf_counter() - start - - @pytest.fixture(scope="function") def agents(session_local, host, models): agent_ids = [] @@ -67,9 +59,9 @@ async def _create_checkins(session_local, models, agent_ids): ) -agent_count = 10 -time_delta = 5 # 17280 checkins per agent per day -days_back = 7 +agent_count = 2 +time_delta = 20 # 4320 checkins per agent per day +days_back = 3 end_time = datetime(2023, 1, 8, tzinfo=timezone.utc) start_time = end_time - timedelta(days=days_back) @@ -88,51 +80,6 @@ async def _create_checkin(session_local, models, agent_id): db_2.add_all(checkins) -@pytest.mark.slow -def test_database_performance_checkins(models, host, agents, session_local): - # logging.basicConfig() - # logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) - # logging.getLogger("sqlalchemy.engine").propagate = True - # print(query.statement.compile(compile_kwargs={"literal_binds": True})) - - with session_local() as db: - asyncio.run(_create_checkins(session_local, models, agents)) - - with timer() as t: - checkins = db.query(models.AgentCheckIn).count() - assert checkins >= (agent_count * 17280 * days_back) - log.info(f"Time to query {checkins} checkins count: {t():0.4f} seconds") - - with timer() as t: - agents = db.query(models.Agent).count() - assert agents >= agent_count - log.info(f"Time to query {agents} agents count: {t():0.4f} seconds") - assert t() < 1 - - with timer() as t: - query = db.query(models.Agent) - query.all() - log.info(f"Time to query {agents} agents: {t():0.4f} seconds") - assert t() < 1 - - with timer() as t: - query = db.query(models.AgentCheckIn).limit(50000) - query.all() - log.info(f"Time to query {checkins} checkins: {t():0.4f} seconds") - assert t() < 6 # noqa: PLR2004 - - agents = db.query(models.Agent).all() - - with timer() as t: - for a in agents: - name = a.name - lastseen_time = a.lastseen_time - stale = a.stale - log.info(f"{name} - {lastseen_time} - {stale}") - log.info(f"Time to query {agents} agents' dynamic fields: {t():0.4f} seconds") - assert t() < 0.1 * agent_count - - def test_get_agent_checkins_agent_not_found(client, admin_auth_header): response = client.get("/api/v2/agents/XYZ123/checkins", headers=admin_auth_header) @@ -153,7 +100,7 @@ def test_get_agent_checkins_with_limit_and_page( checkin_count = 10 assert response.status_code == status.HTTP_200_OK assert len(response.json()["records"]) == checkin_count - assert response.json()["total"] > days_back * 17280 + assert response.json()["total"] > days_back * 4320 assert response.json()["page"] == 1 page1 = response.json()["records"] @@ -166,7 +113,7 @@ def test_get_agent_checkins_with_limit_and_page( page_count = 2 assert response.status_code == status.HTTP_200_OK assert len(response.json()["records"]) == checkin_count - assert response.json()["total"] > days_back * 17280 + assert response.json()["total"] > days_back * 4320 assert response.json()["page"] == page_count page2 = response.json()["records"] @@ -178,18 +125,17 @@ def test_get_agent_checkins_with_limit_and_page( def test_get_agent_checkins_multiple_agents( client, admin_auth_header, agents, session_local, models ): - with_checkins = agents[:3] - asyncio.run(_create_checkins(session_local, models, with_checkins)) + asyncio.run(_create_checkins(session_local, models, agents)) response = client.get( "/api/v2/agents/checkins", headers=admin_auth_header, - params={"agents": with_checkins[:2], "limit": 400000}, + params={"agents": agents, "limit": 400000}, ) assert response.status_code == status.HTTP_200_OK - assert len(response.json()["records"]) == days_back * 17280 * 2 - assert {r["agent_id"] for r in response.json()["records"]} == set(with_checkins[:2]) + assert len(response.json()["records"]) == days_back * 4320 * agent_count + assert {r["agent_id"] for r in response.json()["records"]} == set(agents) @pytest.mark.slow @@ -199,7 +145,7 @@ def test_agent_checkins_aggregate( if empire_config.database.use == "sqlite": pytest.skip("sqlite not supported for checkin aggregation") - asyncio.run(_create_checkins(session_local, models, agents[:3])) + asyncio.run(_create_checkins(session_local, models, agents)) response = client.get( "/api/v2/agents/checkins/aggregate", @@ -209,7 +155,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "day" - assert response.json()["records"][1]["count"] == 17280 * 3 + assert response.json()["records"][1]["count"] == 4320 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -220,7 +166,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "hour" - assert response.json()["records"][1]["count"] == 720 * 3 + assert response.json()["records"][1]["count"] == 180 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -231,7 +177,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "minute" - assert response.json()["records"][1]["count"] == 12 * 3 + assert response.json()["records"][1]["count"] == 3 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -246,7 +192,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "second" - assert response.json()["records"][1]["count"] == 1 * 3 + assert response.json()["records"][1]["count"] == 1 * agent_count # Test start date and end date response = client.get( diff --git a/poetry.lock b/poetry.lock index f30e82dcf..e4e01faf2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1722,7 +1722,10 @@ files = [ [package.dependencies] annotated-types = ">=0.4.0" pydantic-core = "2.20.1" -typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} +typing-extensions = [ + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, +] [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1911,31 +1914,31 @@ files = [ [[package]] name = "pyinstaller" -version = "6.9.0" +version = "6.11.1" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.8" +python-versions = "<3.14,>=3.8" files = [ - {file = "pyinstaller-6.9.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:5ced2e83acf222b936ea94abc5a5cc96588705654b39138af8fb321d9cf2b954"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:f18a3d551834ef8fb7830d48d4cc1527004d0e6b51ded7181e78374ad6111846"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_i686.whl", hash = "sha256:f2fc568de3d6d2a176716a3fc9f20da06d351e8bea5ddd10ecb5659fce3a05b0"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:a0f378f64ad0655d11ade9fde7877e7573fd3d5066231608ce7dfa9040faecdd"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:7bf0c13c5a8560c89540746ae742f4f4b82290e95a6b478374d9f34959fe25d6"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:da994aba14c5686db88796684de265a8665733b4df09b939f7ebdf097d18df72"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:4e3e50743c091a06e6d01c59bdd6d03967b453ee5384a9e790759be4129db4a4"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b041be2fe78da47a269604d62c940d68c62f9a3913bdf64af4123f7689d47099"}, - {file = "pyinstaller-6.9.0-py3-none-win32.whl", hash = "sha256:2bf4de17a1c63c0b797b38e13bfb4d03b5ee7c0a68e28b915a7eaacf6b76087f"}, - {file = "pyinstaller-6.9.0-py3-none-win_amd64.whl", hash = "sha256:43709c70b1da8441a730327a8ed362bfcfdc3d42c1bf89f3e2b0a163cc4e7d33"}, - {file = "pyinstaller-6.9.0-py3-none-win_arm64.whl", hash = "sha256:f15c1ef11ed5ceb32447dfbdab687017d6adbef7fc32aa359d584369bfe56eda"}, - {file = "pyinstaller-6.9.0.tar.gz", hash = "sha256:f4a75c552facc2e2a370f1e422b971b5e5cdb4058ff38cea0235aa21fc0b378f"}, + {file = "pyinstaller-6.11.1-py3-none-macosx_10_13_universal2.whl", hash = "sha256:44e36172de326af6d4e7663b12f71dbd34e2e3e02233e181e457394423daaf03"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6d12c45a29add78039066a53fb05967afaa09a672426072b13816fe7676abfc4"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_i686.whl", hash = "sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:0d6475559c4939f0735122989611d7f739ed3bf02f666ce31022928f7a7e4fda"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:32c742a24fe65d0702958fadf4040f76de85859c26bec0008766e5dbabc5b68f"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:208c0ef6dab0837a0a273ea32d1a3619a208e3d1fe3fec3785eea71a77fd00ce"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:ad84abf465bcda363c1d54eafa76745d77b6a8a713778348377dc98d12a452f7"}, + {file = "pyinstaller-6.11.1-py3-none-win32.whl", hash = "sha256:2e8365276c5131c9bef98e358fbc305e4022db8bedc9df479629d6414021956a"}, + {file = "pyinstaller-6.11.1-py3-none-win_amd64.whl", hash = "sha256:7ac83c0dc0e04357dab98c487e74ad2adb30e7eb186b58157a8faf46f1fa796f"}, + {file = "pyinstaller-6.11.1-py3-none-win_arm64.whl", hash = "sha256:35e6b8077d240600bb309ed68bb0b1453fd2b7ab740b66d000db7abae6244423"}, + {file = "pyinstaller-6.11.1.tar.gz", hash = "sha256:491dfb4d9d5d1d9650d9507daec1ff6829527a254d8e396badd60a0affcb72ef"}, ] [package.dependencies] altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} packaging = ">=22.0" -pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2024.7" +pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2024.9" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" @@ -1945,13 +1948,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.7" +version = "2024.10" description = "Community maintained hooks for PyInstaller" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl", hash = "sha256:8bf0775771fbaf96bcd2f4dfd6f7ae6c1dd1b1efe254c7e50477b3c08e7841d8"}, - {file = "pyinstaller_hooks_contrib-2024.7.tar.gz", hash = "sha256:fd5f37dcf99bece184e40642af88be16a9b89613ecb958a8bd1136634fc9fac5"}, + {file = "pyinstaller_hooks_contrib-2024.10-py3-none-any.whl", hash = "sha256:ad47db0e153683b4151e10d231cb91f2d93c85079e78d76d9e0f57ac6c8a5e10"}, + {file = "pyinstaller_hooks_contrib-2024.10.tar.gz", hash = "sha256:8a46655e5c5b0186b5e527399118a9b342f10513eb1425c483fa4f6d02e8800c"}, ] [package.dependencies] @@ -3356,5 +3359,5 @@ test = ["pytest"] [metadata] lock-version = "2.0" -python-versions = ">=3.10,<3.13" -content-hash = "3c88435392ff46f7cba202a6eb4633fe996739a004fd3b0fc9bc0113de257c13" +python-versions = ">=3.10,<3.14" +content-hash = "200cdb9e7bcdb6647d2d5906974d8487a66befff4b4c7613b769694aa7bddd4b" diff --git a/pyproject.toml b/pyproject.toml index 23b6c4074..8fe40a7a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "empire-bc-security-fork" -version = "5.11.7" +version = "5.12.0" description = "" authors = ["BC Security "] readme = "README.md" @@ -13,7 +13,7 @@ packages = [ ] [tool.poetry.dependencies] -python = ">=3.10,<3.13" +python = ">=3.10,<3.14" urllib3 = "^2.2.0" requests = "^2.31.0" iptools = "^0.7.0" diff --git a/setup/install.sh b/setup/install.sh index 28c3c4c6a..fb37cd1b2 100755 --- a/setup/install.sh +++ b/setup/install.sh @@ -292,7 +292,7 @@ if ! command_exists pyenv; then apt-get -y install build-essential gdb lcov pkg-config \ libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \ - lzma lzma-dev tk-dev uuid-dev zlib1g-dev + lzma tk-dev uuid-dev zlib1g-dev pyenv install 3.12.6 fi