From 072f5d70a6dfc0d58c27e20f0dbdf0f9e1b27b3b Mon Sep 17 00:00:00 2001 From: Vincent Rose Date: Tue, 12 Nov 2024 17:50:30 -0700 Subject: [PATCH 1/7] [wip] try profiling tests to find out why newer python versions are slower (#935) * add profiling * run slow tests * remove runslow decorator and checkins api tests * remove --runslow * add conftest.py back * only run the checkin tests * use runslow * remove performance test * reduce amount of data in checkin tests and get rid of performance test * run all tests * remove loop_factory workaround * 'good enough' * format --- .github/workflows/lint-and-test.yml | 4 +- CHANGELOG.md | 2 + empire/test/conftest.py | 4 +- empire/test/test_agent_checkins_api.py | 82 +++++--------------------- 4 files changed, 20 insertions(+), 72 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 7006ac25c..c4bf2e520 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -85,7 +85,7 @@ jobs: DATABASE_USE=sqlite poetry run pytest . -v --runslow - name: Pytest coverage comment if: ${{ matrix.python-version == '3.12' }} - uses: MishaKav/pytest-coverage-comment@v1.1.53 + uses: MishaKav/pytest-coverage-comment@v1.1.52 with: pytest-coverage-path: ./pytest-coverage.txt junitxml-path: ./pytest.xml @@ -139,7 +139,7 @@ jobs: # To save CI time, only run these tests when the install script or deps changed - name: Get changed files using defaults id: changed-files - uses: tj-actions/changed-files@v45.0.4 + uses: tj-actions/changed-files@v45.0.3 - name: Build images if: contains(steps.changed-files.outputs.modified_files, 'setup/install.sh') || contains(steps.changed-files.outputs.modified_files, 'poetry.lock') run: docker compose -f .github/install_tests/docker-compose-install-tests.yml build --parallel ${{ join(matrix.images, ' ') }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e5e338d..5de5c8984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Reduce the check-in tests that were adding an unncessary amount of time to the CI + ## [5.11.7] - 2024-11-11 - Fix arm installs by installing dotnet and powershell manually diff --git a/empire/test/conftest.py b/empire/test/conftest.py index 5b034935c..6c96d9461 100644 --- a/empire/test/conftest.py +++ b/empire/test/conftest.py @@ -1,4 +1,3 @@ -import asyncio import os import shutil import sys @@ -57,7 +56,8 @@ def client(): # fix for pycharm debugger # https://stackoverflow.com/a/77926544/5849681 - yield TestClient(app, backend_options={"loop_factory": asyncio.new_event_loop}) + # yield TestClient(app, backend_options={"loop_factory": asyncio.new_event_loop}) + yield TestClient(app) from empire.server.server import main diff --git a/empire/test/test_agent_checkins_api.py b/empire/test/test_agent_checkins_api.py index 19ae2c595..5fa044405 100644 --- a/empire/test/test_agent_checkins_api.py +++ b/empire/test/test_agent_checkins_api.py @@ -1,7 +1,5 @@ import asyncio import logging -import time -from contextlib import contextmanager from datetime import datetime, timedelta, timezone import pytest @@ -10,12 +8,6 @@ log = logging.getLogger(__name__) -@contextmanager -def timer(): - start = time.perf_counter() - yield lambda: time.perf_counter() - start - - @pytest.fixture(scope="function") def agents(session_local, host, models): agent_ids = [] @@ -67,9 +59,9 @@ async def _create_checkins(session_local, models, agent_ids): ) -agent_count = 10 -time_delta = 5 # 17280 checkins per agent per day -days_back = 7 +agent_count = 2 +time_delta = 20 # 4320 checkins per agent per day +days_back = 3 end_time = datetime(2023, 1, 8, tzinfo=timezone.utc) start_time = end_time - timedelta(days=days_back) @@ -88,51 +80,6 @@ async def _create_checkin(session_local, models, agent_id): db_2.add_all(checkins) -@pytest.mark.slow -def test_database_performance_checkins(models, host, agents, session_local): - # logging.basicConfig() - # logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) - # logging.getLogger("sqlalchemy.engine").propagate = True - # print(query.statement.compile(compile_kwargs={"literal_binds": True})) - - with session_local() as db: - asyncio.run(_create_checkins(session_local, models, agents)) - - with timer() as t: - checkins = db.query(models.AgentCheckIn).count() - assert checkins >= (agent_count * 17280 * days_back) - log.info(f"Time to query {checkins} checkins count: {t():0.4f} seconds") - - with timer() as t: - agents = db.query(models.Agent).count() - assert agents >= agent_count - log.info(f"Time to query {agents} agents count: {t():0.4f} seconds") - assert t() < 1 - - with timer() as t: - query = db.query(models.Agent) - query.all() - log.info(f"Time to query {agents} agents: {t():0.4f} seconds") - assert t() < 1 - - with timer() as t: - query = db.query(models.AgentCheckIn).limit(50000) - query.all() - log.info(f"Time to query {checkins} checkins: {t():0.4f} seconds") - assert t() < 6 # noqa: PLR2004 - - agents = db.query(models.Agent).all() - - with timer() as t: - for a in agents: - name = a.name - lastseen_time = a.lastseen_time - stale = a.stale - log.info(f"{name} - {lastseen_time} - {stale}") - log.info(f"Time to query {agents} agents' dynamic fields: {t():0.4f} seconds") - assert t() < 0.1 * agent_count - - def test_get_agent_checkins_agent_not_found(client, admin_auth_header): response = client.get("/api/v2/agents/XYZ123/checkins", headers=admin_auth_header) @@ -153,7 +100,7 @@ def test_get_agent_checkins_with_limit_and_page( checkin_count = 10 assert response.status_code == status.HTTP_200_OK assert len(response.json()["records"]) == checkin_count - assert response.json()["total"] > days_back * 17280 + assert response.json()["total"] > days_back * 4320 assert response.json()["page"] == 1 page1 = response.json()["records"] @@ -166,7 +113,7 @@ def test_get_agent_checkins_with_limit_and_page( page_count = 2 assert response.status_code == status.HTTP_200_OK assert len(response.json()["records"]) == checkin_count - assert response.json()["total"] > days_back * 17280 + assert response.json()["total"] > days_back * 4320 assert response.json()["page"] == page_count page2 = response.json()["records"] @@ -178,18 +125,17 @@ def test_get_agent_checkins_with_limit_and_page( def test_get_agent_checkins_multiple_agents( client, admin_auth_header, agents, session_local, models ): - with_checkins = agents[:3] - asyncio.run(_create_checkins(session_local, models, with_checkins)) + asyncio.run(_create_checkins(session_local, models, agents)) response = client.get( "/api/v2/agents/checkins", headers=admin_auth_header, - params={"agents": with_checkins[:2], "limit": 400000}, + params={"agents": agents, "limit": 400000}, ) assert response.status_code == status.HTTP_200_OK - assert len(response.json()["records"]) == days_back * 17280 * 2 - assert {r["agent_id"] for r in response.json()["records"]} == set(with_checkins[:2]) + assert len(response.json()["records"]) == days_back * 4320 * agent_count + assert {r["agent_id"] for r in response.json()["records"]} == set(agents) @pytest.mark.slow @@ -199,7 +145,7 @@ def test_agent_checkins_aggregate( if empire_config.database.use == "sqlite": pytest.skip("sqlite not supported for checkin aggregation") - asyncio.run(_create_checkins(session_local, models, agents[:3])) + asyncio.run(_create_checkins(session_local, models, agents)) response = client.get( "/api/v2/agents/checkins/aggregate", @@ -209,7 +155,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "day" - assert response.json()["records"][1]["count"] == 17280 * 3 + assert response.json()["records"][1]["count"] == 4320 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -220,7 +166,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "hour" - assert response.json()["records"][1]["count"] == 720 * 3 + assert response.json()["records"][1]["count"] == 180 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -231,7 +177,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "minute" - assert response.json()["records"][1]["count"] == 12 * 3 + assert response.json()["records"][1]["count"] == 3 * agent_count response = client.get( "/api/v2/agents/checkins/aggregate", @@ -246,7 +192,7 @@ def test_agent_checkins_aggregate( assert response.status_code == status.HTTP_200_OK assert response.elapsed.total_seconds() < 5 # noqa: PLR2004 assert response.json()["bucket_size"] == "second" - assert response.json()["records"][1]["count"] == 1 * 3 + assert response.json()["records"][1]["count"] == 1 * agent_count # Test start date and end date response = client.get( From 26bc10c12b1cf417c0e3ca23bc4cd51ac60bb27b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 21:24:44 -0700 Subject: [PATCH 2/7] Bump tj-actions/changed-files from 45.0.3 to 45.0.4 (#940) Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 45.0.3 to 45.0.4. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v45.0.3...v45.0.4) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index c4bf2e520..012f58fa1 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -139,7 +139,7 @@ jobs: # To save CI time, only run these tests when the install script or deps changed - name: Get changed files using defaults id: changed-files - uses: tj-actions/changed-files@v45.0.3 + uses: tj-actions/changed-files@v45.0.4 - name: Build images if: contains(steps.changed-files.outputs.modified_files, 'setup/install.sh') || contains(steps.changed-files.outputs.modified_files, 'poetry.lock') run: docker compose -f .github/install_tests/docker-compose-install-tests.yml build --parallel ${{ join(matrix.images, ' ') }} From 6fc3f4dd70e3d6c8ddf612d4f9ecfe1999a85b36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 21:24:57 -0700 Subject: [PATCH 3/7] Bump MishaKav/pytest-coverage-comment from 1.1.52 to 1.1.53 (#939) Bumps [MishaKav/pytest-coverage-comment](https://github.com/mishakav/pytest-coverage-comment) from 1.1.52 to 1.1.53. - [Release notes](https://github.com/mishakav/pytest-coverage-comment/releases) - [Changelog](https://github.com/MishaKav/pytest-coverage-comment/blob/main/CHANGELOG.md) - [Commits](https://github.com/mishakav/pytest-coverage-comment/compare/v1.1.52...v1.1.53) --- updated-dependencies: - dependency-name: MishaKav/pytest-coverage-comment dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 012f58fa1..7006ac25c 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -85,7 +85,7 @@ jobs: DATABASE_USE=sqlite poetry run pytest . -v --runslow - name: Pytest coverage comment if: ${{ matrix.python-version == '3.12' }} - uses: MishaKav/pytest-coverage-comment@v1.1.52 + uses: MishaKav/pytest-coverage-comment@v1.1.53 with: pytest-coverage-path: ./pytest-coverage.txt junitxml-path: ./pytest.xml From 63ddcf04e0f434cc8f02e4bf45c53679543abc91 Mon Sep 17 00:00:00 2001 From: Vincent Rose Date: Wed, 4 Dec 2024 21:56:06 -0700 Subject: [PATCH 4/7] Allow Python 3.13 (#944) --- CHANGELOG.md | 1 + poetry.lock | 49 ++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de5c8984..1d0e9fae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Reduce the check-in tests that were adding an unncessary amount of time to the CI +- Allow Python 3.13 to be used ## [5.11.7] - 2024-11-11 diff --git a/poetry.lock b/poetry.lock index f30e82dcf..e4e01faf2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1722,7 +1722,10 @@ files = [ [package.dependencies] annotated-types = ">=0.4.0" pydantic-core = "2.20.1" -typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} +typing-extensions = [ + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, +] [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1911,31 +1914,31 @@ files = [ [[package]] name = "pyinstaller" -version = "6.9.0" +version = "6.11.1" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.8" +python-versions = "<3.14,>=3.8" files = [ - {file = "pyinstaller-6.9.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:5ced2e83acf222b936ea94abc5a5cc96588705654b39138af8fb321d9cf2b954"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:f18a3d551834ef8fb7830d48d4cc1527004d0e6b51ded7181e78374ad6111846"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_i686.whl", hash = "sha256:f2fc568de3d6d2a176716a3fc9f20da06d351e8bea5ddd10ecb5659fce3a05b0"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:a0f378f64ad0655d11ade9fde7877e7573fd3d5066231608ce7dfa9040faecdd"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:7bf0c13c5a8560c89540746ae742f4f4b82290e95a6b478374d9f34959fe25d6"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:da994aba14c5686db88796684de265a8665733b4df09b939f7ebdf097d18df72"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:4e3e50743c091a06e6d01c59bdd6d03967b453ee5384a9e790759be4129db4a4"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b041be2fe78da47a269604d62c940d68c62f9a3913bdf64af4123f7689d47099"}, - {file = "pyinstaller-6.9.0-py3-none-win32.whl", hash = "sha256:2bf4de17a1c63c0b797b38e13bfb4d03b5ee7c0a68e28b915a7eaacf6b76087f"}, - {file = "pyinstaller-6.9.0-py3-none-win_amd64.whl", hash = "sha256:43709c70b1da8441a730327a8ed362bfcfdc3d42c1bf89f3e2b0a163cc4e7d33"}, - {file = "pyinstaller-6.9.0-py3-none-win_arm64.whl", hash = "sha256:f15c1ef11ed5ceb32447dfbdab687017d6adbef7fc32aa359d584369bfe56eda"}, - {file = "pyinstaller-6.9.0.tar.gz", hash = "sha256:f4a75c552facc2e2a370f1e422b971b5e5cdb4058ff38cea0235aa21fc0b378f"}, + {file = "pyinstaller-6.11.1-py3-none-macosx_10_13_universal2.whl", hash = "sha256:44e36172de326af6d4e7663b12f71dbd34e2e3e02233e181e457394423daaf03"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6d12c45a29add78039066a53fb05967afaa09a672426072b13816fe7676abfc4"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_i686.whl", hash = "sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:0d6475559c4939f0735122989611d7f739ed3bf02f666ce31022928f7a7e4fda"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:32c742a24fe65d0702958fadf4040f76de85859c26bec0008766e5dbabc5b68f"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:208c0ef6dab0837a0a273ea32d1a3619a208e3d1fe3fec3785eea71a77fd00ce"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:ad84abf465bcda363c1d54eafa76745d77b6a8a713778348377dc98d12a452f7"}, + {file = "pyinstaller-6.11.1-py3-none-win32.whl", hash = "sha256:2e8365276c5131c9bef98e358fbc305e4022db8bedc9df479629d6414021956a"}, + {file = "pyinstaller-6.11.1-py3-none-win_amd64.whl", hash = "sha256:7ac83c0dc0e04357dab98c487e74ad2adb30e7eb186b58157a8faf46f1fa796f"}, + {file = "pyinstaller-6.11.1-py3-none-win_arm64.whl", hash = "sha256:35e6b8077d240600bb309ed68bb0b1453fd2b7ab740b66d000db7abae6244423"}, + {file = "pyinstaller-6.11.1.tar.gz", hash = "sha256:491dfb4d9d5d1d9650d9507daec1ff6829527a254d8e396badd60a0affcb72ef"}, ] [package.dependencies] altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} packaging = ">=22.0" -pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2024.7" +pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2024.9" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" @@ -1945,13 +1948,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.7" +version = "2024.10" description = "Community maintained hooks for PyInstaller" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl", hash = "sha256:8bf0775771fbaf96bcd2f4dfd6f7ae6c1dd1b1efe254c7e50477b3c08e7841d8"}, - {file = "pyinstaller_hooks_contrib-2024.7.tar.gz", hash = "sha256:fd5f37dcf99bece184e40642af88be16a9b89613ecb958a8bd1136634fc9fac5"}, + {file = "pyinstaller_hooks_contrib-2024.10-py3-none-any.whl", hash = "sha256:ad47db0e153683b4151e10d231cb91f2d93c85079e78d76d9e0f57ac6c8a5e10"}, + {file = "pyinstaller_hooks_contrib-2024.10.tar.gz", hash = "sha256:8a46655e5c5b0186b5e527399118a9b342f10513eb1425c483fa4f6d02e8800c"}, ] [package.dependencies] @@ -3356,5 +3359,5 @@ test = ["pytest"] [metadata] lock-version = "2.0" -python-versions = ">=3.10,<3.13" -content-hash = "3c88435392ff46f7cba202a6eb4633fe996739a004fd3b0fc9bc0113de257c13" +python-versions = ">=3.10,<3.14" +content-hash = "200cdb9e7bcdb6647d2d5906974d8487a66befff4b4c7613b769694aa7bddd4b" diff --git a/pyproject.toml b/pyproject.toml index 23b6c4074..022747979 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ ] [tool.poetry.dependencies] -python = ">=3.10,<3.13" +python = ">=3.10,<3.14" urllib3 = "^2.2.0" requests = "^2.31.0" iptools = "^0.7.0" From 7daaed5c0a8cc6d10b982bc7113b690d55ea83b2 Mon Sep 17 00:00:00 2001 From: Anthony Rose <20302208+Cx01N@users.noreply.github.com> Date: Sat, 14 Dec 2024 12:39:53 -0500 Subject: [PATCH 5/7] Remove lzma-dev from python install (#953) --- setup/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/install.sh b/setup/install.sh index 7eeeace3c..0e1970123 100755 --- a/setup/install.sh +++ b/setup/install.sh @@ -295,7 +295,7 @@ if ! command_exists pyenv; then apt-get -y install build-essential gdb lcov pkg-config \ libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \ - lzma lzma-dev tk-dev uuid-dev zlib1g-dev + lzma tk-dev uuid-dev zlib1g-dev pyenv install 3.12.6 fi From c4ff98326d880a2706b4349e3894853e7ab36980 Mon Sep 17 00:00:00 2001 From: Antonio Date: Sat, 7 Dec 2024 23:20:23 +0100 Subject: [PATCH 6/7] Support Empire for system-wide deployment (#757) --- .github/cst-config-docker.yaml | 3 - .../cst-config-install-base.yaml | 3 - CHANGELOG.md | 4 + Dockerfile | 4 +- docs/quickstart/configuration/client.md | 2 + docs/quickstart/configuration/server.md | 2 + empire.py | 5 +- empire/client/src/EmpireCliConfig.py | 5 +- empire/config_manager.py | 122 ++++++++++++++++++ empire/server/core/config.py | 44 +++++-- empire/server/modules/bof/nanodump.py | 3 +- empire/server/server.py | 18 +-- empire/test/test_zz_reset.py | 8 +- setup/install.sh | 3 - 14 files changed, 173 insertions(+), 53 deletions(-) create mode 100644 empire/config_manager.py diff --git a/.github/cst-config-docker.yaml b/.github/cst-config-docker.yaml index 195bdef20..7f268c26f 100644 --- a/.github/cst-config-docker.yaml +++ b/.github/cst-config-docker.yaml @@ -48,9 +48,6 @@ fileExistenceTests: - name: 'profiles' path: '/empire/empire/server/data/profiles/' shouldExist: true - - name: 'invoke obfuscation' - path: '/usr/local/share/powershell/Modules/Invoke-Obfuscation/' - shouldExist: true - name: 'sharpire' path: '/empire/empire/server/csharp/Covenant/Data/ReferenceSourceLibraries/Sharpire' shouldExist: true diff --git a/.github/install_tests/cst-config-install-base.yaml b/.github/install_tests/cst-config-install-base.yaml index d79acc82a..61bcbb77d 100644 --- a/.github/install_tests/cst-config-install-base.yaml +++ b/.github/install_tests/cst-config-install-base.yaml @@ -71,9 +71,6 @@ fileExistenceTests: - name: 'profiles' path: '/empire/empire/server/data/profiles/' shouldExist: true - - name: 'invoke obfuscation' - path: '/usr/local/share/powershell/Modules/Invoke-Obfuscation/' - shouldExist: true - name: 'sharpire' path: '/empire/empire/server/csharp/Covenant/Data/ReferenceSourceLibraries/Sharpire' shouldExist: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d0e9fae1..0cb14d88a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reduce the check-in tests that were adding an unncessary amount of time to the CI - Allow Python 3.13 to be used +- Fix python install +- Support Empire for system-wide deployment (@D3vil0p3r) +- Paths specified in config.yaml where user does not have write permission will be fallback to ~/.empire directory and config.yaml updated as well (@D3vil0p3r) +- Invoke-Obfuscation is no longer copied to /usr/local/share ## [5.11.7] - 2024-11-11 diff --git a/Dockerfile b/Dockerfile index 61fdf3e43..65fd89ca9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,9 +59,7 @@ RUN poetry config virtualenvs.create false && \ COPY . /empire -RUN mkdir -p /usr/local/share/powershell/Modules && \ - cp -r ./empire/server/data/Invoke-Obfuscation /usr/local/share/powershell/Modules && \ - rm -rf /empire/empire/server/data/empire* +RUN rm -rf /empire/empire/server/data/empire* RUN sed -i 's/use: mysql/use: sqlite/g' empire/server/config.yaml && \ sed -i 's/auto_update: true/auto_update: false/g' empire/server/config.yaml diff --git a/docs/quickstart/configuration/client.md b/docs/quickstart/configuration/client.md index 22d0f294f..393f10ee9 100644 --- a/docs/quickstart/configuration/client.md +++ b/docs/quickstart/configuration/client.md @@ -5,6 +5,8 @@ The Client configuration is managed via [empire/client/config.yaml](https://github.com/BC-SECURITY/Empire/blob/master/empire/client/config.yaml). +Once launched, Empire checks for user write permissions on paths specified in `config.yaml`. If the current user does not have write permissions on these paths, `~/.empire` will be set as fallback parent directory and the configuration file will be updated as well. + * **servers** - The servers block is meant to give the user the ability to set up frequently used Empire servers. If a server is listed in this block then when connecting to the server they need only type: `connect -c localhost`. diff --git a/docs/quickstart/configuration/server.md b/docs/quickstart/configuration/server.md index e6c0a9c44..4e00381ee 100644 --- a/docs/quickstart/configuration/server.md +++ b/docs/quickstart/configuration/server.md @@ -2,6 +2,8 @@ The Server configuration is managed via [empire/server/config.yaml](https://github.com/BC-SECURITY/Empire/blob/master/empire/client/config.yaml). +Once launched, Empire checks for user write permissions on paths specified in `config.yaml`. If the current user does not have write permissions on these paths, `~/.empire` will be set as fallback parent directory and the configuration file will be updated as well. + * **suppress-self-cert-warning** - Suppress the http warnings when launching an Empire instance that uses a self-signed cert. * **api** - Configure the RESTful API. This includes the port to run the API on, as well as the path for the SSL certificates. If `empire-priv.key` and `empire-chain.pem` are not found in this directory, self-signed certs will be generated. diff --git a/empire.py b/empire.py index a9687dec4..9c199f719 100644 --- a/empire.py +++ b/empire.py @@ -2,10 +2,11 @@ import sys -from empire import arguments +from empire import arguments, config_manager if __name__ == "__main__": args = arguments.args + config_manager.config_init() if args.subparser_name == "server": from empire.server import server @@ -16,7 +17,7 @@ from empire.scripts.sync_starkiller import sync_starkiller - with open("empire/server/config.yaml") as f: + with open(config_manager.CONFIG_SERVER_PATH) as f: config = yaml.safe_load(f) sync_starkiller(config) diff --git a/empire/client/src/EmpireCliConfig.py b/empire/client/src/EmpireCliConfig.py index 10efeaa28..9029ebb71 100644 --- a/empire/client/src/EmpireCliConfig.py +++ b/empire/client/src/EmpireCliConfig.py @@ -3,6 +3,8 @@ import yaml +from empire import config_manager + log = logging.getLogger(__name__) @@ -15,7 +17,8 @@ def __init__(self): self.set_yaml(location) if len(self.yaml.items()) == 0: log.info("Loading default config") - self.set_yaml("./empire/client/config.yaml") + self.set_yaml(config_manager.CONFIG_CLIENT_PATH) + config_manager.check_config_permission(self.yaml, "client") def set_yaml(self, location: str): try: diff --git a/empire/config_manager.py b/empire/config_manager.py new file mode 100644 index 000000000..5f172133c --- /dev/null +++ b/empire/config_manager.py @@ -0,0 +1,122 @@ +import logging +import os +import shutil +from pathlib import Path + +import yaml + +log = logging.getLogger(__name__) + +user_home = Path.home() +SOURCE_CONFIG_CLIENT = Path("empire/client/config.yaml") +SOURCE_CONFIG_SERVER = Path("empire/server/config.yaml") +CONFIG_DIR = user_home / ".empire" +CONFIG_CLIENT_PATH = CONFIG_DIR / "client" / "config.yaml" +CONFIG_SERVER_PATH = CONFIG_DIR / "server" / "config.yaml" + + +def config_init(): + CONFIG_CLIENT_PATH.parent.mkdir(parents=True, exist_ok=True) + CONFIG_SERVER_PATH.parent.mkdir(parents=True, exist_ok=True) + + if not CONFIG_CLIENT_PATH.exists(): + shutil.copy(SOURCE_CONFIG_CLIENT, CONFIG_CLIENT_PATH) + log.info(f"Copied {SOURCE_CONFIG_CLIENT} to {CONFIG_CLIENT_PATH}") + else: + log.info(f"{CONFIG_CLIENT_PATH} already exists.") + + if not CONFIG_SERVER_PATH.exists(): + shutil.copy(SOURCE_CONFIG_SERVER, CONFIG_SERVER_PATH) + log.info(f"Copied {SOURCE_CONFIG_SERVER} to {CONFIG_SERVER_PATH}") + else: + log.info(f"{CONFIG_SERVER_PATH} already exists.") + + +def check_config_permission(config_dict: dict, config_type: str): + """ + Check if the specified directories in config.yaml are writable. If not, switches to a fallback directory. + Handles both server and client configurations. + + Args: + config_dict (dict): The configuration dictionary loaded from YAML. + config_type (str): The type of configuration ("server" or "client"). + """ + # Define paths to check based on config type + if config_type == "server": + paths_to_check = { + ("api", "cert_path"): config_dict.get("api", {}).get("cert_path"), + ("database", "sqlite", "location"): config_dict.get("database", {}) + .get("sqlite", {}) + .get("location"), + ("starkiller", "directory"): config_dict.get("starkiller", {}).get( + "directory" + ), + ("logging", "directory"): config_dict.get("logging", {}).get("directory"), + ("debug", "last_task", "file"): config_dict.get("debug", {}) + .get("last_task", {}) + .get("file"), + ("directories", "downloads"): config_dict.get("directories", {}).get( + "downloads" + ), + } + config_path = CONFIG_SERVER_PATH # Use the server config path + + elif config_type == "client": + paths_to_check = { + ("logging", "directory"): config_dict.get("logging", {}).get("directory"), + ("directories", "downloads"): config_dict.get("directories", {}).get( + "downloads" + ), + ("directories", "generated-stagers"): config_dict.get( + "directories", {} + ).get("generated-stagers"), + } + config_path = CONFIG_CLIENT_PATH # Use the client config path + + else: + raise ValueError("Invalid config_type. Expected 'server' or 'client'.") + + # Check permissions and update paths as needed + for keys, dir_path in paths_to_check.items(): + if dir_path is None: + continue + + current_dir = dir_path + while current_dir and not os.path.exists(current_dir): + current_dir = os.path.dirname(current_dir) + + if not os.access(current_dir, os.W_OK): + log.info( + "No write permission for %s. Switching to fallback directory.", + current_dir, + ) + user_home = Path.home() + fallback_dir = os.path.join( + user_home, ".empire", str(current_dir).removeprefix("empire/") + ) + + # Update the directory in config_dict + target = config_dict # target is a reference to config_dict + for key in keys[:-1]: + target = target[key] + target[keys[-1]] = fallback_dir + + log.info( + "Updated %s to fallback directory: %s", "->".join(keys), fallback_dir + ) + + # Write the updated configuration back to the correct YAML file + with open(config_path, "w") as config_file: + yaml.safe_dump(paths2str(config_dict), config_file) + + return config_dict + + +def paths2str(data): + if isinstance(data, dict): + return {key: paths2str(value) for key, value in data.items()} + if isinstance(data, list): + return [paths2str(item) for item in data] + if isinstance(data, Path): + return str(data) + return data diff --git a/empire/server/core/config.py b/empire/server/core/config.py index 539c56c87..46eea3f51 100644 --- a/empire/server/core/config.py +++ b/empire/server/core/config.py @@ -5,6 +5,8 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, field_validator +from empire import config_manager + log = logging.getLogger(__name__) @@ -74,9 +76,9 @@ def __getitem__(self, key): class DirectoriesConfig(EmpireBaseModel): - downloads: Path - module_source: Path - obfuscated_module_source: Path + downloads: Path = Path("empire/server/downloads") + module_source: Path = Path("empire/server/modules") + obfuscated_module_source: Path = Path("empire/server/data/obfuscated_module_source") class LoggingConfig(EmpireBaseModel): @@ -99,17 +101,26 @@ class EmpireConfig(EmpireBaseModel): alias="supress-self-cert-warning", default=True ) api: ApiConfig | None = ApiConfig() - starkiller: StarkillerConfig - submodules: SubmodulesConfig - database: DatabaseConfig + starkiller: StarkillerConfig = StarkillerConfig() + submodules: SubmodulesConfig = SubmodulesConfig() + database: DatabaseConfig = DatabaseConfig( + sqlite=SQLiteDatabaseConfig(), + mysql=MySQLDatabaseConfig(), + defaults=DatabaseDefaultsConfig(), + ) plugins: dict[str, dict[str, str]] = {} - directories: DirectoriesConfig - logging: LoggingConfig - debug: DebugConfig + directories: DirectoriesConfig = DirectoriesConfig() + logging: LoggingConfig = LoggingConfig() + debug: DebugConfig = DebugConfig(last_task=LastTaskConfig()) model_config = ConfigDict(extra="allow") - def __init__(self, config_dict: dict): + def __init__(self, config_dict: dict | None = None): + if config_dict is None: + config_dict = {} + if not isinstance(config_dict, dict): + raise ValueError("config_dict must be a dictionary") + super().__init__(**config_dict) # For backwards compatibility self.yaml = config_dict @@ -126,13 +137,18 @@ def set_yaml(location: str): log.warning(exc) -config_dict = {} +config_dict = EmpireConfig().model_dump() if "--config" in sys.argv: location = sys.argv[sys.argv.index("--config") + 1] log.info(f"Loading config from {location}") - config_dict = set_yaml(location) -if len(config_dict.items()) == 0: + loaded_config = set_yaml(location) + if loaded_config: + config_dict = loaded_config +elif config_manager.CONFIG_SERVER_PATH.exists(): log.info("Loading default config") - config_dict = set_yaml("./empire/server/config.yaml") + loaded_config = set_yaml(config_manager.CONFIG_SERVER_PATH) + if loaded_config: + config_dict = loaded_config + config_dict = config_manager.check_config_permission(config_dict, "server") empire_config = EmpireConfig(config_dict) diff --git a/empire/server/modules/bof/nanodump.py b/empire/server/modules/bof/nanodump.py index 30d635115..c214234b8 100644 --- a/empire/server/modules/bof/nanodump.py +++ b/empire/server/modules/bof/nanodump.py @@ -16,8 +16,7 @@ def generate( module=module, params=params, obfuscate=obfuscate ) - for name in params: - value = params[name] + for name, value in params.items(): if name == "write": if value != "": dump_path = value diff --git a/empire/server/server.py b/empire/server/server.py index bfeb69cd2..01287e692 100755 --- a/empire/server/server.py +++ b/empire/server/server.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import logging import os -import pathlib import pwd import shutil import signal @@ -67,10 +66,6 @@ def setup_logging(args): CSHARP_DIR_BASE = os.path.join(os.path.dirname(__file__), "csharp/Covenant") -INVOKE_OBFS_SRC_DIR_BASE = os.path.join( - os.path.dirname(__file__), "data/Invoke-Obfuscation" -) -INVOKE_OBFS_DST_DIR_BASE = "/usr/local/share/powershell/Modules/Invoke-Obfuscation" def reset(): @@ -94,16 +89,6 @@ def reset(): if os.path.exists(empire_config.starkiller.directory): shutil.rmtree(empire_config.starkiller.directory) - # invoke obfuscation - if os.path.exists(f"{INVOKE_OBFS_DST_DIR_BASE}"): - shutil.rmtree(INVOKE_OBFS_DST_DIR_BASE) - pathlib.Path(pathlib.Path(INVOKE_OBFS_SRC_DIR_BASE).parent).mkdir( - parents=True, exist_ok=True - ) - shutil.copytree( - INVOKE_OBFS_SRC_DIR_BASE, INVOKE_OBFS_DST_DIR_BASE, dirs_exist_ok=True - ) - file_util.remove_file("data/sessions.csv") file_util.remove_file("data/credentials.csv") file_util.remove_file("data/master.log") @@ -144,6 +129,9 @@ def check_submodules(): def fetch_submodules(): + if not os.path.exists(Path(".git")): + log.info("No .git directory found. Skipping submodule fetch.") + return command = ["git", "submodule", "update", "--init", "--recursive"] run_as_user(command) diff --git a/empire/test/test_zz_reset.py b/empire/test/test_zz_reset.py index a3c7dbfe1..da047377d 100644 --- a/empire/test/test_zz_reset.py +++ b/empire/test/test_zz_reset.py @@ -39,8 +39,6 @@ def test_reset_server(monkeypatch, tmp_path, default_argv, server_config_dict): 1. Deletes the sqlite db. Don't need to test mysql atm. 2. Deletes the downloads dir contents 3. Deletes the csharp generated files - 4. Deletes the obfuscated modules - 5. Deletes / Copies invoke obfuscation """ monkeypatch.setattr("builtins.input", lambda _: "y") sys.argv = [*default_argv.copy(), "--reset"] @@ -64,9 +62,8 @@ def test_reset_server(monkeypatch, tmp_path, default_argv, server_config_dict): for f in download_files: assert Path(downloads_dir + f[0]).exists() - # Change the csharp and Invoke-Obfuscation dir so we don't delete real files. + # Change the csharp dir so we don't delete real files. csharp_dir = tmp_path / "empire/server/data/csharp" - invoke_obfs_dir = tmp_path / "powershell/Modules/Invoke-Obfuscation" # Write files to csharp_dir csharp_files = [ @@ -105,7 +102,6 @@ def test_reset_server(monkeypatch, tmp_path, default_argv, server_config_dict): assert Path(server_config_dict["database"]["location"]).exists() server.CSHARP_DIR_BASE = csharp_dir - server.INVOKE_OBFS_DST_DIR_BASE = invoke_obfs_dir with pytest.raises(SystemExit): server.run(args) @@ -126,8 +122,6 @@ def test_reset_server(monkeypatch, tmp_path, default_argv, server_config_dict): csharp_dir / "Data/Tasks/CSharp/Compiled/netcoreapp3.0" / f[0] ).exists() - assert Path(invoke_obfs_dir / "Invoke-Obfuscation.ps1").exists() - if server_config_dict.get("database", {}).get("type") == "sqlite": assert not Path(server_config_dict["database"]["location"]).exists() diff --git a/setup/install.sh b/setup/install.sh index 0e1970123..fb37cd1b2 100755 --- a/setup/install.sh +++ b/setup/install.sh @@ -40,9 +40,6 @@ function install_powershell() { sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 sudo chmod +x /opt/microsoft/powershell/7/pwsh sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh - - sudo mkdir -p /usr/local/share/powershell/Modules - sudo cp -r "$PARENT_PATH"/empire/server/data/Invoke-Obfuscation /usr/local/share/powershell/Modules } function install_mysql() { From 42c168a494845c1b1241a3e39009c6b027b58403 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Sat, 14 Dec 2024 18:39:47 +0000 Subject: [PATCH 7/7] Prepare release 5.12.0 private --- CHANGELOG.md | 6 +++++- empire/server/common/empire.py | 2 +- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cb14d88a..3a888ac39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.12.0] - 2024-12-14 + - Reduce the check-in tests that were adding an unncessary amount of time to the CI - Allow Python 3.13 to be used - Fix python install @@ -937,7 +939,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated shellcoderdi to newest version (@Cx01N) - Added a Nim launcher (@Hubbl3) -[Unreleased]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.7...HEAD +[Unreleased]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.12.0...HEAD + +[5.12.0]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.7...v5.12.0 [5.11.7]: https://github.com/BC-SECURITY/Empire-Sponsors/compare/v5.11.6...v5.11.7 diff --git a/empire/server/common/empire.py b/empire/server/common/empire.py index c4495f2c0..f77cfafaa 100755 --- a/empire/server/common/empire.py +++ b/empire/server/common/empire.py @@ -38,7 +38,7 @@ from . import agents, credentials, listeners, stagers -VERSION = "5.11.7 BC Security Fork" +VERSION = "5.12.0 BC Security Fork" log = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 022747979..8fe40a7a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "empire-bc-security-fork" -version = "5.11.7" +version = "5.12.0" description = "" authors = ["BC Security "] readme = "README.md"