diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d278d9a..f3ad74e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,9 +14,15 @@ jobs: prevent: true fail-on-error: true allowed-ips: > + 3.216.34.172 + 3.228.146.75 + 18.206.20.10 + 18.210.197.188 18.215.138.58 34.194.164.123 + 34.205.13.154 44.196.175.70 + 44.205.64.79 52.1.184.176 52.3.144.121 54.165.156.197 @@ -64,9 +70,15 @@ jobs: prevent: true fail-on-error: true allowed-ips: > + 3.216.34.172 + 3.228.146.75 + 18.206.20.10 + 18.210.197.188 18.215.138.58 34.194.164.123 + 34.205.13.154 44.196.175.70 + 44.205.64.79 52.1.184.176 52.3.144.121 54.165.156.197 @@ -85,7 +97,6 @@ jobs: with: ref: ${{ github.ref }} - - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test_pr.yml b/.github/workflows/test_pr.yml index 482b990..64103e6 100644 --- a/.github/workflows/test_pr.yml +++ b/.github/workflows/test_pr.yml @@ -57,6 +57,8 @@ jobs: fi exit 0 + # A job that runs integration tests in an isolated environment against + # a predefined organization: RavenIntegrationTests test_raven: runs-on: ubuntu-latest steps: @@ -65,9 +67,15 @@ jobs: prevent: true fail-on-error: true allowed-ips: > + 3.216.34.172 + 3.228.146.75 + 18.206.20.10 + 18.210.197.188 18.215.138.58 34.194.164.123 + 34.205.13.154 44.196.175.70 + 44.205.64.79 52.1.184.176 52.3.144.121 54.165.156.197 @@ -90,6 +98,8 @@ jobs: run: | make test-build + # A job for testing the setup process and unit tests of RAVEN against + # different versions of Python test_raven_package: runs-on: ubuntu-latest @@ -103,9 +113,15 @@ jobs: prevent: true fail-on-error: true allowed-ips: > + 3.216.34.172 + 3.228.146.75 + 18.206.20.10 + 18.210.197.188 18.215.138.58 34.194.164.123 + 34.205.13.154 44.196.175.70 + 44.205.64.79 52.1.184.176 52.3.144.121 54.165.156.197 @@ -133,10 +149,13 @@ jobs: - name: Setup environment run: make setup - - name: Download Organization + - name: Test Raven env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | raven download org --token $GITHUB_TOKEN --org-name RavenIntegrationTests raven index - raven report --format json | jq > /dev/null \ No newline at end of file + raven report --format json | jq > /dev/null + + - name: Run Unit Tests + run: pytest -v tests/unit \ No newline at end of file diff --git a/.gitignore b/.gitignore index b9106f3..b0b06c0 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,7 @@ ipython_config.py # Local debugging - Run tests outside containers temp_test_raven.py +temp/ # pyenv .python-version diff --git a/Makefile b/Makefile index 9653ab8..795acee 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,4 @@ test-build: test-run: @echo "DO NOT USE DIRECTLY; PLEASE USE: make test-build" @echo "Running Tests..." - @python3 test_raven.py \ No newline at end of file + @pytest -v tests \ No newline at end of file diff --git a/deployment/test.dockerfile b/deployment/test.dockerfile index 7acdf5a..4d03399 100644 --- a/deployment/test.dockerfile +++ b/deployment/test.dockerfile @@ -11,9 +11,6 @@ COPY Makefile requirements.txt /raven/ COPY src /raven/src COPY tests /raven/tests -# Move the main test file to the Root directory -RUN mv /raven/tests/test_raven.py /raven/test_raven.py - # Install any needed packages specified in requirements.txt RUN pip3 install -r requirements.txt diff --git a/src/common/utils.py b/src/common/utils.py index a29e5f6..c6c3671 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -121,3 +121,7 @@ def is_url_contains_a_token(url) -> bool: def str_to_bool(s: str) -> bool: return bool(int(s)) + + +def raw_str_to_bool(s: str) -> bool: + return True if s == "true" else False diff --git a/src/workflow_components/composite_action.py b/src/workflow_components/composite_action.py index 5da1d26..31093d6 100644 --- a/src/workflow_components/composite_action.py +++ b/src/workflow_components/composite_action.py @@ -8,6 +8,7 @@ from src.common.utils import ( get_dependencies_in_code, convert_dict_to_list, + raw_str_to_bool, ) from src.workflow_components.dependency import UsesString, UsesStringType @@ -27,6 +28,43 @@ def get_or_create_composite_action(path: str) -> "CompositeAction": return obj +class CompositeActionInput(GraphObject): + __primarykey__ = "_id" + + _id = Property() + name = Property() + default = Property() + description = Property() + required = Property() + url = Property() + path = Property() + action = RelatedTo("CompositeAction") + + def __init__(self, _id: str, path: str): + self._id = _id + self.path = path + + @staticmethod + def from_dict(obj_dict) -> "CompositeActionInput": + i = CompositeActionInput( + _id=obj_dict["_id"], + path=obj_dict["path"], + ) + + i.name = obj_dict["name"] + i.url = obj_dict["url"] + + if "default" in obj_dict: + i.default = obj_dict["default"] + + if "description" in obj_dict: + i.description = obj_dict["description"] + + i.required = raw_str_to_bool(obj_dict.get("required", "false")) + + return i + + class CompositeActionStep(GraphObject): __primarykey__ = "_id" @@ -91,12 +129,12 @@ class CompositeAction(GraphObject): _id = Property() name = Property() path = Property() - inputs = Property() using = Property() image = Property() url = Property() is_public = Property() + inputs = RelatedTo(CompositeActionInput) steps = RelatedTo(CompositeActionStep) def __init__(self, name: Optional[str], path: str): @@ -111,7 +149,12 @@ def from_dict(obj_dict) -> "CompositeAction": ca.url = obj_dict["url"] ca.is_public = obj_dict["is_public"] if "inputs" in obj_dict: - ca.inputs = list(obj_dict["inputs"].keys()) + for name, input in obj_dict["inputs"].items(): + input["_id"] = md5(f"{ca._id}_{name}".encode()).hexdigest() + input["name"] = name + input["url"] = ca.url + input["path"] = ca.path + ca.inputs.add(CompositeActionInput.from_dict(input)) if "runs" in obj_dict: d_runs = obj_dict["runs"] diff --git a/tests/integration/integration_consts.py b/tests/integration/integration_consts.py index cb0d590..cce0a9a 100644 --- a/tests/integration/integration_consts.py +++ b/tests/integration/integration_consts.py @@ -48,3 +48,6 @@ }, }, ] + +START_NODE_INDEX = 0 +DEST_NODE_INDEX = 2 diff --git a/tests/integration/structures_json/demo-index.json b/tests/integration/structures_json/demo-index.json index 1541fcf..9afc4b2 100644 --- a/tests/integration/structures_json/demo-index.json +++ b/tests/integration/structures_json/demo-index.json @@ -1,193 +1,593 @@ { - "nodes": [{ - "path": "actions/checkout", - "using": "node20", - "inputs": ["repository", "ref", "token", "ssh-key", "ssh-known-hosts", "ssh-strict", "persist-credentials", "path", "clean", "filter", "sparse-checkout", "sparse-checkout-cone-mode", "fetch-depth", "fetch-tags", "show-progress", "lfs", "submodules", "set-safe-directory", "github-server-url"], - "name": "Checkout", - "is_public": true, - "_id": "d35e7df441120da9624b8c11e36151be", - "url": "https://github.com/actions/checkout/tree/main/action.yml", - "labels": ["CompositeAction"] - }, { - "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", - "name": "run", - "is_public": true, - "_id": "d142a4f8137ab3cb3316a8d45a8406a7", - "trigger": ["workflow_dispatch"], - "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Workflow"] - }, { - "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", - "machine": ["ubuntu-latest"], - "name": "demo_test", - "_id": "77ef9503ba52c912546d499c4e05f557", - "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Job"] - }, { - "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", - "ref": "v4", - "uses": "actions/checkout@v4", - "_id": "83b34c58d9a3220059a67e3671ff97f2", - "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", - "name": "PrintEnv", - "run": "printenv", - "_id": "4a6d2fcdd26ea9a8949530cd9b61eb9b", - "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", - "name": "run", - "is_public": true, - "_id": "c79850ebc886b835b32def127e189b8a", - "trigger": ["workflow_dispatch"], - "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Workflow"] - }, { - "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", - "machine": ["ubuntu-latest"], - "name": "demo_test", - "_id": "fecf155691da067f7da9db40a66bcb5b", - "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Job"] - }, { - "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", - "ref": "v4", - "uses": "actions/checkout@v4", - "_id": "8f8cdae1f2db6f6feb3aed267ef5bc34", - "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", - "name": "PrintEnv", - "run": "printenv", - "_id": "25b1ab6f07c18556ab4833f77c576e32", - "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", - "name": "run", - "is_public": true, - "_id": "b195de8206ae6d901a448576cdba759f", - "trigger": ["workflow_dispatch"], - "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Workflow"] - }, { - "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", - "machine": ["ubuntu-latest"], - "name": "demo_test", - "_id": "672e94911c5b6ad0af76aa61413d236c", - "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Job"] - }, { - "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", - "ref": "v4", - "uses": "actions/checkout@v4", - "_id": "77508179e2034107334d369fef96c1dc", - "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", - "name": "PrintEnv", - "run": "printenv", - "_id": "7a09d64801de95b2f329cdda5838df97", - "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", - "name": "run", - "is_public": true, - "_id": "22137cee99760506f369934d49f719f8", - "trigger": ["workflow_dispatch"], - "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Workflow"] - }, { - "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", - "machine": ["ubuntu-latest"], - "name": "demo_test", - "_id": "ec3b1d0fd5ec71b9f43547f026f579fd", - "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Job"] - }, { - "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", - "ref": "v4", - "uses": "actions/checkout@v4", - "_id": "c1306c77e3af9cdedcedee69c59c96c1", - "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }, { - "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", - "name": "PrintEnv", - "run": "printenv", - "_id": "6f76bec5bf9d4e683ef736d3ffe8b842", - "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", - "labels": ["Step"] - }], - "relationships": [{ - "start_node": "83b34c58d9a3220059a67e3671ff97f2", - "type": "ACTION", - "end_node": "d35e7df441120da9624b8c11e36151be" - }, { - "start_node": "77ef9503ba52c912546d499c4e05f557", - "type": "STEPS", - "end_node": "83b34c58d9a3220059a67e3671ff97f2" - }, { - "start_node": "77ef9503ba52c912546d499c4e05f557", - "type": "STEPS", - "end_node": "4a6d2fcdd26ea9a8949530cd9b61eb9b" - }, { - "start_node": "d142a4f8137ab3cb3316a8d45a8406a7", - "type": "JOBS", - "end_node": "77ef9503ba52c912546d499c4e05f557" - }, { - "start_node": "8f8cdae1f2db6f6feb3aed267ef5bc34", - "type": "ACTION", - "end_node": "d35e7df441120da9624b8c11e36151be" - }, { - "start_node": "fecf155691da067f7da9db40a66bcb5b", - "type": "STEPS", - "end_node": "8f8cdae1f2db6f6feb3aed267ef5bc34" - }, { - "start_node": "fecf155691da067f7da9db40a66bcb5b", - "type": "STEPS", - "end_node": "25b1ab6f07c18556ab4833f77c576e32" - }, { - "start_node": "c79850ebc886b835b32def127e189b8a", - "type": "JOBS", - "end_node": "fecf155691da067f7da9db40a66bcb5b" - }, { - "start_node": "77508179e2034107334d369fef96c1dc", - "type": "ACTION", - "end_node": "d35e7df441120da9624b8c11e36151be" - }, { - "start_node": "672e94911c5b6ad0af76aa61413d236c", - "type": "STEPS", - "end_node": "77508179e2034107334d369fef96c1dc" - }, { - "start_node": "672e94911c5b6ad0af76aa61413d236c", - "type": "STEPS", - "end_node": "7a09d64801de95b2f329cdda5838df97" - }, { - "start_node": "b195de8206ae6d901a448576cdba759f", - "type": "JOBS", - "end_node": "672e94911c5b6ad0af76aa61413d236c" - }, { - "start_node": "c1306c77e3af9cdedcedee69c59c96c1", - "type": "ACTION", - "end_node": "d35e7df441120da9624b8c11e36151be" - }, { - "start_node": "ec3b1d0fd5ec71b9f43547f026f579fd", - "type": "STEPS", - "end_node": "c1306c77e3af9cdedcedee69c59c96c1" - }, { - "start_node": "ec3b1d0fd5ec71b9f43547f026f579fd", - "type": "STEPS", - "end_node": "6f76bec5bf9d4e683ef736d3ffe8b842" - }, { - "start_node": "22137cee99760506f369934d49f719f8", - "type": "JOBS", - "end_node": "ec3b1d0fd5ec71b9f43547f026f579fd" - }] + "nodes": [ + { + "path": "actions/checkout", + "using": "node20", + "name": "Checkout", + "is_public": true, + "_id": "d35e7df441120da9624b8c11e36151be", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeAction" + ] + }, + { + "path": "actions/checkout", + "default": "${{ github.repository }}", + "name": "repository", + "description": "Repository name with owner. For example, actions/checkout", + "_id": "775c9f7b8b404a9df37b16c9d4d8f336", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "ref", + "description": "The branch, tag or SHA to checkout. When checking out the repository that triggered a workflow, this defaults to the reference or SHA for that event. Otherwise, uses the default branch.\n", + "_id": "6bf31cd9bee2b2c9a0fd9a8d3c578903", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "${{ github.token }}", + "name": "token", + "description": "Personal access token (PAT) used to fetch the repository. The PAT is configured with the local git config, which enables your scripts to run authenticated git commands. The post-job step removes the PAT.\n\nWe recommend using a service account with the least permissions necessary. Also when generating a new PAT, select the least scopes necessary.\n\n[Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets)\n", + "_id": "5f4fa35f28767a9155a5813b2cc934d5", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "ssh-key", + "description": "SSH key used to fetch the repository. The SSH key is configured with the local git config, which enables your scripts to run authenticated git commands. The post-job step removes the SSH key.\n\nWe recommend using a service account with the least permissions necessary.\n\n[Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets)\n", + "_id": "c53d6fe7a19a576c8bfefe6cfa80870b", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "ssh-known-hosts", + "description": "Known hosts in addition to the user and global host key database. The public SSH keys for a host may be obtained using the utility `ssh-keyscan`. For example, `ssh-keyscan github.com`. The public key for github.com is always implicitly added.\n", + "_id": "145d5c9a78c4d7564f7b79ec495b6ff2", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "ssh-strict", + "description": "Whether to perform strict host key checking. When true, adds the options `StrictHostKeyChecking=yes` and `CheckHostIP=no` to the SSH command line. Use the input `ssh-known-hosts` to configure additional hosts.\n", + "_id": "ec56ff9d7ca004ba974a2fca1b01f2c5", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "persist-credentials", + "description": "Whether to configure the token or SSH key with the local git config", + "_id": "fe19be86972043c79c7ad0f618559ade", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "path", + "description": "Relative path under $GITHUB_WORKSPACE to place the repository", + "_id": "3470907d9833db0f15032d1aadd1dbc2", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "clean", + "description": "Whether to execute `git clean -ffdx && git reset --hard HEAD` before fetching", + "_id": "a8d8ef152d8986d297b84bc2faf66d1e", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "filter", + "description": "Partially clone against a given filter. Overrides sparse-checkout if set.\n", + "_id": "974202e17cde42b352e662c9c55df9ff", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "sparse-checkout", + "description": "Do a sparse checkout on given patterns. Each pattern should be separated with new lines.\n", + "_id": "c626f387aeac44aa82b5e27f4cbaca54", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "sparse-checkout-cone-mode", + "description": "Specifies whether to use cone-mode when doing a sparse checkout.\n", + "_id": "90aa405b4e4278ac32892e1e1756b807", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": 1, + "name": "fetch-depth", + "description": "Number of commits to fetch. 0 indicates all history for all branches and tags.", + "_id": "00fb2bfc353337457e5be8ea01e27984", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "false", + "name": "fetch-tags", + "description": "Whether to fetch tags, even if fetch-depth > 0.", + "_id": "a32f4c2bbd5fb33b56c7b2634b343e62", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "show-progress", + "description": "Whether to show progress status output when fetching.", + "_id": "c871387fa99df881290dbd906bae83a6", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "false", + "name": "lfs", + "description": "Whether to download Git-LFS files", + "_id": "99f82cfda3b119fcf2b38ebc651dc0a8", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "false", + "name": "submodules", + "description": "Whether to checkout submodules: `true` to checkout submodules or `recursive` to recursively checkout submodules.\n\nWhen the `ssh-key` input is not provided, SSH URLs beginning with `git@github.com:` are converted to HTTPS.\n", + "_id": "7cdf94b14af56d70417e7b6d92ff316e", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "default": "true", + "name": "set-safe-directory", + "description": "Add repository path as safe.directory for Git global config by running `git config --global --add safe.directory `", + "_id": "98aaa1fa6903184fa592300457546c9e", + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "required": false, + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "actions/checkout", + "name": "github-server-url", + "description": "The base URL for the GitHub instance that you are trying to clone from, will use environment defaults to fetch from the same instance that the workflow is running from unless specified. Example URLs are https://github.com or https://my-ghes-server.example.com", + "_id": "e8f0d587842e43432e7c96ddcf2ef5e8", + "required": false, + "url": "https://github.com/actions/checkout/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, + { + "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", + "is_public": true, + "name": "run", + "_id": "d142a4f8137ab3cb3316a8d45a8406a7", + "trigger": [ + "workflow_dispatch" + ], + "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Workflow" + ] + }, + { + "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", + "machine": [ + "ubuntu-latest" + ], + "name": "demo_test", + "_id": "77ef9503ba52c912546d499c4e05f557", + "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Job" + ] + }, + { + "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", + "ref": "v4", + "uses": "actions/checkout@v4", + "_id": "83b34c58d9a3220059a67e3671ff97f2", + "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-2/.github/workflows/demo-workflow.yml", + "name": "PrintEnv", + "run": "printenv", + "_id": "4a6d2fcdd26ea9a8949530cd9b61eb9b", + "url": "https://github.com/RavenIntegrationTests/Demo-2/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", + "is_public": true, + "name": "run", + "_id": "c79850ebc886b835b32def127e189b8a", + "trigger": [ + "workflow_dispatch" + ], + "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Workflow" + ] + }, + { + "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", + "machine": [ + "ubuntu-latest" + ], + "name": "demo_test", + "_id": "fecf155691da067f7da9db40a66bcb5b", + "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Job" + ] + }, + { + "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", + "ref": "v4", + "uses": "actions/checkout@v4", + "_id": "8f8cdae1f2db6f6feb3aed267ef5bc34", + "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-4/.github/workflows/demo-workflow.yml", + "name": "PrintEnv", + "run": "printenv", + "_id": "25b1ab6f07c18556ab4833f77c576e32", + "url": "https://github.com/RavenIntegrationTests/Demo-4/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", + "is_public": true, + "name": "run", + "_id": "b195de8206ae6d901a448576cdba759f", + "trigger": [ + "workflow_dispatch" + ], + "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Workflow" + ] + }, + { + "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", + "machine": [ + "ubuntu-latest" + ], + "name": "demo_test", + "_id": "672e94911c5b6ad0af76aa61413d236c", + "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Job" + ] + }, + { + "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", + "ref": "v4", + "uses": "actions/checkout@v4", + "_id": "77508179e2034107334d369fef96c1dc", + "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-3/.github/workflows/demo-workflow.yml", + "name": "PrintEnv", + "run": "printenv", + "_id": "7a09d64801de95b2f329cdda5838df97", + "url": "https://github.com/RavenIntegrationTests/Demo-3/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", + "is_public": true, + "name": "run", + "_id": "22137cee99760506f369934d49f719f8", + "trigger": [ + "workflow_dispatch" + ], + "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Workflow" + ] + }, + { + "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", + "machine": [ + "ubuntu-latest" + ], + "name": "demo_test", + "_id": "ec3b1d0fd5ec71b9f43547f026f579fd", + "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Job" + ] + }, + { + "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", + "ref": "v4", + "uses": "actions/checkout@v4", + "_id": "c1306c77e3af9cdedcedee69c59c96c1", + "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + }, + { + "path": "RavenIntegrationTests/Demo-1/.github/workflows/demo-workflow.yml", + "name": "PrintEnv", + "run": "printenv", + "_id": "6f76bec5bf9d4e683ef736d3ffe8b842", + "url": "https://github.com/RavenIntegrationTests/Demo-1/tree/main/.github/workflows/demo-workflow.yml", + "labels": [ + "Step" + ] + } + ], + "relationships": [ + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "775c9f7b8b404a9df37b16c9d4d8f336" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "6bf31cd9bee2b2c9a0fd9a8d3c578903" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "5f4fa35f28767a9155a5813b2cc934d5" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "c53d6fe7a19a576c8bfefe6cfa80870b" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "145d5c9a78c4d7564f7b79ec495b6ff2" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "ec56ff9d7ca004ba974a2fca1b01f2c5" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "fe19be86972043c79c7ad0f618559ade" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "3470907d9833db0f15032d1aadd1dbc2" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "a8d8ef152d8986d297b84bc2faf66d1e" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "974202e17cde42b352e662c9c55df9ff" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "c626f387aeac44aa82b5e27f4cbaca54" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "90aa405b4e4278ac32892e1e1756b807" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "00fb2bfc353337457e5be8ea01e27984" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "a32f4c2bbd5fb33b56c7b2634b343e62" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "c871387fa99df881290dbd906bae83a6" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "99f82cfda3b119fcf2b38ebc651dc0a8" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "7cdf94b14af56d70417e7b6d92ff316e" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "98aaa1fa6903184fa592300457546c9e" + }, + { + "start_node": "d35e7df441120da9624b8c11e36151be", + "type": "INPUTS", + "end_node": "e8f0d587842e43432e7c96ddcf2ef5e8" + }, + { + "start_node": "83b34c58d9a3220059a67e3671ff97f2", + "type": "ACTION", + "end_node": "d35e7df441120da9624b8c11e36151be" + }, + { + "start_node": "77ef9503ba52c912546d499c4e05f557", + "type": "STEPS", + "end_node": "83b34c58d9a3220059a67e3671ff97f2" + }, + { + "start_node": "77ef9503ba52c912546d499c4e05f557", + "type": "STEPS", + "end_node": "4a6d2fcdd26ea9a8949530cd9b61eb9b" + }, + { + "start_node": "d142a4f8137ab3cb3316a8d45a8406a7", + "type": "JOBS", + "end_node": "77ef9503ba52c912546d499c4e05f557" + }, + { + "start_node": "8f8cdae1f2db6f6feb3aed267ef5bc34", + "type": "ACTION", + "end_node": "d35e7df441120da9624b8c11e36151be" + }, + { + "start_node": "fecf155691da067f7da9db40a66bcb5b", + "type": "STEPS", + "end_node": "8f8cdae1f2db6f6feb3aed267ef5bc34" + }, + { + "start_node": "fecf155691da067f7da9db40a66bcb5b", + "type": "STEPS", + "end_node": "25b1ab6f07c18556ab4833f77c576e32" + }, + { + "start_node": "c79850ebc886b835b32def127e189b8a", + "type": "JOBS", + "end_node": "fecf155691da067f7da9db40a66bcb5b" + }, + { + "start_node": "77508179e2034107334d369fef96c1dc", + "type": "ACTION", + "end_node": "d35e7df441120da9624b8c11e36151be" + }, + { + "start_node": "672e94911c5b6ad0af76aa61413d236c", + "type": "STEPS", + "end_node": "77508179e2034107334d369fef96c1dc" + }, + { + "start_node": "672e94911c5b6ad0af76aa61413d236c", + "type": "STEPS", + "end_node": "7a09d64801de95b2f329cdda5838df97" + }, + { + "start_node": "b195de8206ae6d901a448576cdba759f", + "type": "JOBS", + "end_node": "672e94911c5b6ad0af76aa61413d236c" + }, + { + "start_node": "c1306c77e3af9cdedcedee69c59c96c1", + "type": "ACTION", + "end_node": "d35e7df441120da9624b8c11e36151be" + }, + { + "start_node": "ec3b1d0fd5ec71b9f43547f026f579fd", + "type": "STEPS", + "end_node": "c1306c77e3af9cdedcedee69c59c96c1" + }, + { + "start_node": "ec3b1d0fd5ec71b9f43547f026f579fd", + "type": "STEPS", + "end_node": "6f76bec5bf9d4e683ef736d3ffe8b842" + }, + { + "start_node": "22137cee99760506f369934d49f719f8", + "type": "JOBS", + "end_node": "ec3b1d0fd5ec71b9f43547f026f579fd" + } + ] } \ No newline at end of file diff --git a/tests/integration/structures_json/integration-1.json b/tests/integration/structures_json/integration-1.json index 2dd2d66..360998d 100644 --- a/tests/integration/structures_json/integration-1.json +++ b/tests/integration/structures_json/integration-1.json @@ -3,9 +3,6 @@ { "path": "RavenIntegrationTests/CompositeAction-Mock", "using": "composite", - "inputs": [ - "param1" - ], "name": "Example composite GitHub action", "is_public": true, "_id": "0cedc4763c9b12640b59748858f52ecb", @@ -14,6 +11,18 @@ "CompositeAction" ] }, + { + "path": "RavenIntegrationTests/CompositeAction-Mock", + "default": "true", + "name": "param1", + "description": "Input parameter placeholder", + "_id": "e0e8b6ca1b9aab6e0dd0e1fe70c88a08", + "required": true, + "url": "https://github.com/RavenIntegrationTests/CompositeAction-Mock/tree/main/action.yml", + "labels": [ + "CompositeActionInput" + ] + }, { "path": "RavenIntegrationTests/CompositeAction-Mock", "shell": "bash", @@ -49,7 +58,9 @@ }, { "path": "RavenIntegrationTests/Integration-1/.github/workflows/integration-workflow.yml", - "machine": ["ubuntu-latest"], + "machine": [ + "ubuntu-latest" + ], "name": "first_job", "_id": "2007449e2ba101423871ac669de5b750", "url": "https://github.com/RavenIntegrationTests/Integration-1/tree/main/.github/workflows/integration-workflow.yml", @@ -79,7 +90,9 @@ }, { "path": "RavenIntegrationTests/Integration-1/.github/workflows/integration-workflow.yml", - "machine": ["ubuntu-latest"], + "machine": [ + "ubuntu-latest" + ], "name": "second_job", "_id": "57e4ebfad3aa1f852f256d59d7c7e982", "url": "https://github.com/RavenIntegrationTests/Integration-1/tree/main/.github/workflows/integration-workflow.yml", @@ -108,6 +121,11 @@ } ], "relationships": [ + { + "start_node": "0cedc4763c9b12640b59748858f52ecb", + "type": "INPUTS", + "end_node": "e0e8b6ca1b9aab6e0dd0e1fe70c88a08" + }, { "start_node": "db50f26195dcd8ca600a046df26f0a3a", "type": "USING_PARAM", diff --git a/tests/integration/test_graph_structures.py b/tests/integration/test_graph_structures.py index bf8b889..07009c6 100644 --- a/tests/integration/test_graph_structures.py +++ b/tests/integration/test_graph_structures.py @@ -4,6 +4,7 @@ assert_graph_structures, ) from tests.integration.integration_consts import TESTS_CONFIGS +from tests.tests_init import init_integration_env def test_graph_structure() -> None: @@ -11,6 +12,7 @@ def test_graph_structure() -> None: Tests the graph structure of the integration tests. It will loop over each test config dictionary on TESTS_CONFIGS list and assert the graph structure is as expected. """ + init_integration_env() for test_config in TESTS_CONFIGS: print( f"{Fore.CYAN}Running integration test: {test_config['test_name']}.{Style.RESET_ALL}" diff --git a/tests/test_raven.py b/tests/tests_init.py similarity index 73% rename from tests/test_raven.py rename to tests/tests_init.py index c873b3a..cad5d0d 100644 --- a/tests/test_raven.py +++ b/tests/tests_init.py @@ -1,10 +1,13 @@ from os import getenv -import pytest -import src.logger.log as log +from src.config.config import load_downloader_config, load_indexer_config from src.downloader.download import download_org_workflows_and_actions from src.indexer.index import index_downloaded_workflows_and_actions -from src.config.config import load_downloader_config, load_indexer_config -from tests.integration.test_graph_structures import test_graph_structure + + +def init_integration_env(): + load_integration_tests_config() + download_org_workflows_and_actions() + index_downloaded_workflows_and_actions() def load_integration_tests_config() -> None: @@ -13,6 +16,8 @@ def load_integration_tests_config() -> None: "debug": False, "token": getenv("GITHUB_TOKEN"), "org_name": ["RavenIntegrationTests"], + "redis_host": "raven-redis-test", + "redis_port": 6379, "clean_redis": False, } ) @@ -30,22 +35,3 @@ def load_integration_tests_config() -> None: "clean_neo4j": False, } ) - - -def init_env(): - load_integration_tests_config() - download_org_workflows_and_actions() - index_downloaded_workflows_and_actions() - - -def test(): - log.info("[x] Starting unit testing") - pytest.main(["-v", "tests/unit"]) - - log.info("[x] Starting Integration testing") - init_env() - test_graph_structure() - - -if __name__ == "__main__": - test() diff --git a/tests/unit/test_composite_action.py b/tests/unit/test_composite_action.py index 47d599d..1312973 100644 --- a/tests/unit/test_composite_action.py +++ b/tests/unit/test_composite_action.py @@ -1,5 +1,5 @@ import src.workflow_components.composite_action as composite_action -from tests.utils import load_test_config +from tests.utils import load_test_config, assert_action_inputs load_test_config() @@ -12,6 +12,7 @@ def test_composite_action_from_dist_node(): "token": { "description": "The GitHub authentication token", "default": "${{ github.token }}", + "required": True, }, "repository": { "description": "The target GitHub repository", @@ -39,13 +40,14 @@ def test_composite_action_from_dist_node(): assert ca.name == ca_d["name"] assert ca.path == ca_d["path"] - assert ca.inputs == list(ca_d["inputs"].keys()) assert ca.using == "node16" assert ca.url == ca_d["url"] assert ca.is_public == ca_d["is_public"] assert ca.image is None assert len(ca.steps) == 0 + assert_action_inputs(ca, ca_d) + def test_composite_action_from_dict_dockerfile(): ca_d = { @@ -70,13 +72,14 @@ def test_composite_action_from_dict_dockerfile(): assert ca.name == ca_d["name"] assert ca.path == ca_d["path"] - assert ca.inputs == list(ca_d["inputs"].keys()) assert ca.using == "docker" assert ca.image == "Dockerfile" assert ca.url == ca_d["url"] assert ca.is_public == ca_d["is_public"] assert len(ca.steps) == 0 + assert_action_inputs(ca, ca_d) + def test_composite_action_from_dict_image(): ca_d = { @@ -105,13 +108,14 @@ def test_composite_action_from_dict_image(): assert ca.name == ca_d["name"] assert ca.path == ca_d["path"] - assert ca.inputs == list(ca_d["inputs"].keys()) assert ca.using == "docker" assert ca.url == ca_d["url"] assert ca.is_public == ca_d["is_public"] assert ca.image == "docker://ghcr.io/calibreapp/image-actions/image-actions:main" assert len(ca.steps) == 0 + assert_action_inputs(ca, ca_d) + def test_composite_action_from_dict_steps(): ca_d = { @@ -150,13 +154,14 @@ def test_composite_action_from_dict_steps(): assert ca.name == ca_d["name"] assert ca.path == ca_d["path"] - assert ca.inputs == list(ca_d["inputs"].keys()) assert ca.using == "composite" assert ca.url == ca_d["url"] assert ca.is_public == ca_d["is_public"] assert ca.image is None assert len(ca.steps) == 1 + assert_action_inputs(ca, ca_d) + def test_composite_action_step_from_dict_run(): step_d = { diff --git a/tests/utils.py b/tests/utils.py index 4a01a35..991a138 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,11 @@ from py2neo.ogm import GraphObject import json - from src.config.config import Config +from src.workflow_components.composite_action import CompositeAction from typing import Tuple, List, Dict, Optional +from tests.integration.integration_consts import START_NODE_INDEX, DEST_NODE_INDEX +from src.common.utils import raw_str_to_bool +from hashlib import md5 class GraphDbMock(object): @@ -130,3 +133,41 @@ def assert_graph_structures(graph_structure: Dict, snapshot_path: str) -> None: assert ( relationship == graph_relations[snapshot_relations.index(relationship)] ), f"Properties of relationships on the same index of graph and snapshot is not equal\n\n{get_dicts_differences(relationship, graph_relations[snapshot_relations.index(relationship)])}\nIn snapshot:\n{relationship}\nIn graph:\n{graph_relations[snapshot_relations.index(relationship)]}" + + +def assert_action_inputs(ca: CompositeAction, ca_d: Dict): + """ + This function asserts that the action inputs are equal to those in the JSON file. + Each composite action is connected to multiple action inputs. + Each input contains different properties such as name, default, description, and required. + + Using `ca.inputs.triples()`, we are iterating over all the inputs of the composite action. + For each input, we check the following: + 1) The ID, name, and URL of the composite action are equal to the ID, name, and URL of the input. + 2) The id of the composite action input is the md5 hash of the composite action id and the input name. + 3) Check that the default, description, and required properties are equal to those in the JSON file. + + Each input is a tuple containing a source node (that will always be the composite action in this case) + the relation type and the destination node - will be the different action inputs. + """ + for input in ca.inputs.triples(): + ca_d_input = ca_d["inputs"][input[2].name] + + assert input[START_NODE_INDEX]._id == ca._id + assert input[DEST_NODE_INDEX].name == ca_d_input["name"] + assert input[DEST_NODE_INDEX].url == ca_d["url"] + assert ( + input[DEST_NODE_INDEX]._id + == md5(f"{ca._id}_{ca_d_input.get('name')}".encode()).hexdigest() + ) + + if "required" in ca_d_input: + assert input[DEST_NODE_INDEX].required == raw_str_to_bool( + ca_d_input["required"] + ) + + if "default" in ca_d_input: + assert input[DEST_NODE_INDEX].default == ca_d_input["default"] + + if "description" in ca_d_input: + assert input[DEST_NODE_INDEX].description == ca_d_input["description"]