-
Notifications
You must be signed in to change notification settings - Fork 81
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add environment integration tests (#1334)
### Feature or Bugfix
Feature
### Detail
- small refactor of existing test code
- add some environment queries with polling logic
- add environment fixtures with safe create and delete
- run tests in parallel
- new testdata format...
``` {
"users": {
"testUserTenant": {
"username": "testUserTenant",
"password": "...",
"groups": [
"DAAdministrators"
]
}
"envs": {
"session_env1": {
"accountId": "123",
"region": "eu-central-1"
}
}
}
### Relates
#1230
### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).
- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
- Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
- Is injection prevented by parametrizing queries?
- Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
- Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
- Do you use a standard proven implementations?
- Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
- Have you used the least-privilege principle? How?
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.- Loading branch information
Showing
15 changed files
with
491 additions
and
177 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,108 +1,134 @@ | ||
| import json | ||
| import logging | ||
| import os | ||
| import sys | ||
| from dataclasses import dataclass | ||
| from typing import List | ||
|
|
||
| import pytest | ||
| from dataclasses_json import dataclass_json | ||
|
|
||
| from tests_new.integration_tests.client import Client | ||
|
|
||
| logging.basicConfig(level=logging.INFO, stream=sys.stdout) | ||
|
|
||
| ## Define user and groups fixtures - We assume they pre-exist in the AWS account. | ||
| pytest_plugins = [ | ||
| 'integration_tests.core.organizations.global_conftest', | ||
| 'integration_tests.core.environment.global_conftest', | ||
| ] | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class User: | ||
| username: str | ||
| password: str | ||
|
|
||
| @staticmethod | ||
| def from_userdata(userdata, username): | ||
| return User(username, userdata[username]['password']) | ||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class Env: | ||
| accountId: str | ||
| region: str | ||
|
|
||
|
|
||
| @dataclass_json | ||
| @dataclass | ||
| class TestData: | ||
| users: dict[str, User] | ||
| envs: dict[str, Env] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session', autouse=True) | ||
| def testdata() -> TestData: | ||
| try: | ||
| return TestData.from_json(open('testdata.json').read()) | ||
| except Exception: | ||
| return TestData.from_json(os.getenv('TESTDATA')) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| def userdata(): | ||
| yield json.loads(os.getenv('USERDATA')) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def userdata(testdata): | ||
| yield testdata.users | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def userTenant(userdata): | ||
| # Existing user with name and password | ||
| # This user needs to belong to `DAAdministrators` group | ||
| yield User.from_userdata(userdata, 'testUserTenant') | ||
| yield userdata['testUserTenant'] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def user1(userdata): | ||
| # Existing user with name and password | ||
| yield User.from_userdata(userdata, 'testUser1') | ||
| yield userdata['testUser1'] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def user2(userdata): | ||
| # Existing user with name and password | ||
| yield User.from_userdata(userdata, 'testUser2') | ||
| yield userdata['testUser2'] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def user3(userdata): | ||
| # Existing user with name and password | ||
| yield User.from_userdata(userdata, 'testUser3') | ||
| yield userdata['testUser3'] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def user4(userdata): | ||
| # Existing user with name and password | ||
| yield User.from_userdata(userdata, 'testUser4') | ||
| yield userdata['testUser4'] | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def group1(): | ||
| # Existing Cognito group with name testGroup1 | ||
| # Add user1 | ||
| yield 'testGroup1' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def group2(): | ||
| # Existing Cognito group with name testGroup2 | ||
| # Add user2 | ||
| yield 'testGroup2' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def group3(): | ||
| # Existing Cognito group with name testGroup3 | ||
| # Add user3 | ||
| yield 'testGroup3' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module', autouse=True) | ||
| @pytest.fixture(scope='session', autouse=True) | ||
| def group4(): | ||
| # Existing Cognito group with name testGroup4 | ||
| # Add user4 | ||
| yield 'testGroup4' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module') | ||
| @pytest.fixture(scope='session') | ||
| def client1(user1) -> Client: | ||
| yield Client(user1.username, user1.password) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module') | ||
| @pytest.fixture(scope='session') | ||
| def client2(user2) -> Client: | ||
| yield Client(user2.username, user2.password) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module') | ||
| @pytest.fixture(scope='session') | ||
| def client3(user3) -> Client: | ||
| yield Client(user3.username, user3.password) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module') | ||
| @pytest.fixture(scope='session') | ||
| def client4(user4) -> Client: | ||
| yield Client(user4.username, user4.password) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='module') | ||
| @pytest.fixture(scope='session') | ||
| def clientTenant(userTenant) -> Client: | ||
| yield Client(userTenant.username, userTenant.password) |
Empty file.
91 changes: 91 additions & 0 deletions
91
tests_new/integration_tests/core/environment/global_conftest.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| import logging | ||
| import re | ||
|
|
||
| import pytest | ||
|
|
||
| from integration_tests.client import GqlError | ||
| from integration_tests.core.environment.queries import create_environment, get_environment, delete_environment | ||
| from integration_tests.utils import poller | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| @poller(check_success=lambda env: not re.match(r'.*IN_PROGRESS|PENDING', env.stack.status, re.IGNORECASE), timeout=600) | ||
| def check_env_ready(client, env_uri): | ||
| env = get_environment(client, env_uri) | ||
| log.info(f'polling {env_uri=}, new {env.stack.status=}') | ||
| return env | ||
|
|
||
|
|
||
| def create_env(client, group, org_uri, account_id, region): | ||
| new_env_uri = create_environment( | ||
| client, name='testEnvA', group=group, organizationUri=org_uri, awsAccountId=account_id, region=region | ||
| )['environmentUri'] | ||
| return check_env_ready(client, new_env_uri) | ||
|
|
||
|
|
||
| def delete_env(client, env_uri): | ||
| check_env_ready(client, env_uri) | ||
| try: | ||
| return delete_environment(client, env_uri) | ||
| except GqlError: | ||
| log.exception('unexpected error when deleting environment') | ||
| return False | ||
|
|
||
|
|
||
| """ | ||
| Session envs persist accross the duration of the whole integ test suite and are meant to make the test suite run faster (env creation takes ~2 mins). | ||
| For this reason they must stay immutable as changes to them will affect the rest of the tests. | ||
| """ | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def session_env1(client1, group1, org1, testdata): | ||
| envdata = testdata.envs['session_env1'] | ||
| env = None | ||
| try: | ||
| env = create_env(client1, group1, org1['organizationUri'], envdata.accountId, envdata.region) | ||
| yield env | ||
| finally: | ||
| if env: | ||
| delete_env(client1, env['environmentUri']) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def session_env2(client1, group1, org1, testdata): | ||
| envdata = testdata.envs['session_env2'] | ||
| env = None | ||
| try: | ||
| env = create_env(client1, group1, org1['organizationUri'], envdata.accountId, envdata.region) | ||
| yield env | ||
| finally: | ||
| if env: | ||
| delete_env(client1, env['environmentUri']) | ||
|
|
||
|
|
||
| """ | ||
| Temp envs will be created and deleted per test, use with caution as they might increase the runtime of the test suite. | ||
| They are suitable to test env mutations. | ||
| """ | ||
|
|
||
|
|
||
| @pytest.fixture(scope='function') | ||
| def temp_env1(client1, group1, org1, testdata): | ||
| envdata = testdata.envs['temp_env1'] | ||
| env = None | ||
| try: | ||
| env = create_env(client1, group1, org1['organizationUri'], envdata.accountId, envdata.region) | ||
| yield env | ||
| finally: | ||
| if env: | ||
| delete_env(client1, env['environmentUri']) | ||
|
|
||
|
|
||
| """ | ||
| Persistent environments must always be present (if not i.e first run they will be created but won't be removed). | ||
| They are suitable for testing backwards compatibility. | ||
| """ | ||
|
|
||
|
|
||
| @pytest.fixture(scope='function') | ||
| def persistent_env1(client1, group1, org1, testdata): ... # TODO |
Oops, something went wrong.