Skip to content

Commit

Permalink
Add tests for no outbound connectivity (#2804)
Browse files Browse the repository at this point in the history 
* Add tests for no outbound connectivity

---------

Co-authored-by: narrieta <narrieta>
  • Loading branch information
@narrieta
narrieta authored Apr 18, 2023
1 parent 7de6133 commit cb56656
Show file tree
Hide file tree
Showing 17 changed files with 432 additions and 74 deletions.
63 changes: 49 additions & 14 deletions tests_e2e/orchestrator/lib/agent_test_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,39 @@
from tests_e2e.tests.lib.agent_test import AgentTest


class TestInfo(object):
"""
Description of a test
"""
# The class that implements the test
test_class: Type[AgentTest]
# If True, an error in the test blocks the execution of the test suite (defaults to False)
blocks_suite: bool

@property
def name(self) -> str:
return self.test_class.__name__

def __str__(self):
return self.name


class TestSuiteInfo(object):
"""
Description of a test suite
"""
# The name of the test suite
name: str
# The tests that comprise the suite
tests: List[Type[AgentTest]]
tests: List[TestInfo]
# Images or image sets (as defined in images.yml) on which the suite must run.
images: List[str]
# The location (region) on which the suite must run; if empty, the suite can run on any location
location: str
# Whether this suite must run on its own test VM
owns_vm: bool
# Customization for the ARM template used when creating the test VM
template: str

def __str__(self):
return self.name
Expand Down Expand Up @@ -139,7 +158,7 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
"""
Loads the description of a TestSuite from its YAML file.
A test suite has 5 properties: name, tests, images, location, and owns-vm. For example:
A test suite has 5 properties: name, tests, images, location, and owns_vm. For example:
name: "AgentBvt"
tests:
Expand All @@ -148,18 +167,22 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
- "bvts/vm_access.py"
images: "endorsed"
location: "eastuseaup"
owns-vm: true
owns_vm: true
* name - A string used to identify the test suite
* tests - A list of the tests in the suite. Each test is specified by the path for its source code relative to
WALinuxAgent/tests_e2e/tests.
* tests - A list of the tests in the suite. Each test can be specified by a string (the path for its source code relative to
WALinuxAgent/tests_e2e/tests), or a dictionary with two items:
* source: the path for its source code relative to WALinuxAgent/tests_e2e/tests
* blocks_suite: [Optional; boolean] If True, a failure on the test will stop execution of the test suite (i.e. the
rest of the tests in the suite will not be executed). By default, a failure on a test does not stop execution of
the test suite.
* images - A string, or a list of strings, specifying the images on which the test suite must be executed. Each value
can be the name of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The
names for images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml.
* location - [Optional; string] If given, the test suite must be executed on that location. If not specified,
or set to an empty string, the test suite will be executed in the default location. This is useful
for test suites that exercise a feature that is enabled only in certain regions.
* owns-vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this
* owns_vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this
value is set to True, new test VMs will be created and will be used exclusively for this test suite.
This is useful for suites that modify the test VMs in such a way that the setup may cause problems
in other test suites (for example, some tests targeted to the HGAP block internet access in order to
Expand All @@ -176,9 +199,15 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
test_suite_info.name = test_suite["name"]

test_suite_info.tests = []
source_files = [AgentTestLoader._SOURCE_CODE_ROOT/"tests"/t for t in test_suite["tests"]]
for f in source_files:
test_suite_info.tests.extend(AgentTestLoader._load_test_classes(f))
for test in test_suite["tests"]:
test_info = TestInfo()
if isinstance(test, str):
test_info.test_class = AgentTestLoader._load_test_class(test)
test_info.blocks_suite = False
else:
test_info.test_class = AgentTestLoader._load_test_class(test["source"])
test_info.blocks_suite = test.get("blocks_suite", False)
test_suite_info.tests.append(test_info)

images = test_suite["images"]
if isinstance(images, str):
Expand All @@ -190,20 +219,26 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
if test_suite_info.location is None:
test_suite_info.location = ""

test_suite_info.owns_vm = "owns-vm" in test_suite and test_suite["owns-vm"]
test_suite_info.owns_vm = "owns_vm" in test_suite and test_suite["owns_vm"]

test_suite_info.template = test_suite.get("template", "")

return test_suite_info

@staticmethod
def _load_test_classes(source_file: Path) -> List[Type[AgentTest]]:
def _load_test_class(relative_path: str) -> Type[AgentTest]:
"""
Takes a 'source_file', which must be a Python module, and returns a list of all the classes derived from AgentTest.
Loads an AgentTest from its source code file, which is given as a path relative to WALinuxAgent/tests_e2e/tests.
"""
spec = importlib.util.spec_from_file_location(f"tests_e2e.tests.{source_file.name}", str(source_file))
full_path: Path = AgentTestLoader._SOURCE_CODE_ROOT/"tests"/relative_path
spec = importlib.util.spec_from_file_location(f"tests_e2e.tests.{relative_path.replace('/', '.').replace('.py', '')}", str(full_path))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# return all the classes in the module that are subclasses of AgentTest but are not AgentTest itself.
return [v for v in module.__dict__.values() if isinstance(v, type) and issubclass(v, AgentTest) and v != AgentTest]
matches = [v for v in module.__dict__.values() if isinstance(v, type) and issubclass(v, AgentTest) and v != AgentTest]
if len(matches) != 1:
raise Exception(f"Error in {full_path} (each test file must contain exactly one class derived from AgentTest)")
return matches[0]

@staticmethod
def _load_images() -> Dict[str, List[VmImageInfo]]:
Expand Down
55 changes: 32 additions & 23 deletions tests_e2e/orchestrator/lib/agent_test_suite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,8 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
with _set_thread_name(suite_full_name): # The thread name is added to the LISA log
log_path: Path = self.context.log_path/f"{suite_full_name}.log"
with set_current_thread_log(log_path):
suite_success: bool = True

try:
log.info("")
log.info("**************************************** %s ****************************************", suite_name)
Expand All @@ -467,69 +469,76 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
summary: List[str] = []

for test in suite.tests:
test_name = test.__name__
test_full_name = f"{suite_name}-{test_name}"
test_full_name = f"{suite_name}-{test.name}"
test_start_time: datetime.datetime = datetime.datetime.now()

log.info("******** Executing %s", test_name)
log.info("******** Executing %s", test.name)
self.context.lisa_log.info("Executing test %s", test_full_name)

try:
test_success: bool = True

test(self.context).run()
try:
test.test_class(self.context).run()

summary.append(f"[Passed] {test_name}")
log.info("******** [Passed] %s", test_name)
summary.append(f"[Passed] {test.name}")
log.info("******** [Passed] %s", test.name)
self.context.lisa_log.info("[Passed] %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
test.name,
TestStatus.PASSED,
test_start_time)
except TestSkipped as e:
summary.append(f"[Skipped] {test_name}")
log.info("******** [Skipped] %s: %s", test_name, e)
summary.append(f"[Skipped] {test.name}")
log.info("******** [Skipped] %s: %s", test.name, e)
self.context.lisa_log.info("******** [Skipped] %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
test.name,
TestStatus.SKIPPED,
test_start_time,
message=str(e))
except AssertionError as e:
success = False
summary.append(f"[Failed] {test_name}")
log.error("******** [Failed] %s: %s", test_name, e)
test_success = False
summary.append(f"[Failed] {test.name}")
log.error("******** [Failed] %s: %s", test.name, e)
self.context.lisa_log.error("******** [Failed] %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
test.name,
TestStatus.FAILED,
test_start_time,
message=str(e))
except: # pylint: disable=bare-except
success = False
summary.append(f"[Error] {test_name}")
log.exception("UNHANDLED EXCEPTION IN %s", test_name)
test_success = False
summary.append(f"[Error] {test.name}")
log.exception("UNHANDLED EXCEPTION IN %s", test.name)
self.context.lisa_log.exception("UNHANDLED EXCEPTION IN %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
test.name,
TestStatus.FAILED,
test_start_time,
message="Unhandled exception.",
add_exception_stack_trace=True)

log.info("")

log.info("********* [Test Results]")
suite_success = suite_success and test_success

if not test_success and test.blocks_suite:
log.warning("%s failed and blocks the suite. Stopping suite execution.", test.name)
break

log.info("")
log.info("******** [Test Results]")
log.info("")
for r in summary:
log.info("\t%s", r)
log.info("")

except: # pylint: disable=bare-except
success = False
suite_success = False
self._report_test_result(
suite_full_name,
suite_name,
Expand All @@ -538,7 +547,7 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
message=f"Unhandled exception while executing test suite {suite_name}.",
add_exception_stack_trace=True)
finally:
if not success:
if not suite_success:
self._mark_log_as_failed()

return success
Expand All @@ -562,7 +571,7 @@ def _check_agent_log(self) -> bool:
# E1133: Non-iterable value self.context.test_suites is used in an iterating context (not-an-iterable)
for suite in self.context.test_suites: # pylint: disable=E1133
for test in suite.tests:
ignore_error_rules.extend(test(self.context).get_ignore_error_rules())
ignore_error_rules.extend(test.test_class(self.context).get_ignore_error_rules())

if len(ignore_error_rules) > 0:
new = []
Expand Down
51 changes: 29 additions & 22 deletions tests_e2e/orchestrator/lib/agent_test_suite_combinator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,36 +218,44 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
else:
vm_size = ""

if suite_info.owns_vm:
# create an environment for exclusive use by this suite
environment_list.append({
# Note: Disabling "W0640: Cell variable 'foo' defined in loop (cell-var-from-loop)". This is a false positive, the closure is OK
# to use, since create_environment() is called within the same iteration of the loop.
# pylint: disable=W0640
def create_environment(env_name: str) -> Dict[str, Any]:
tags = {}
if suite_info.template != '':
tags["templates"] = suite_info.template
return {
"c_marketplace_image": marketplace_image,
"c_cloud": self.runbook.cloud,
"c_location": location,
"c_vm_size": vm_size,
"c_vhd": vhd,
"c_test_suites": [suite_info],
"c_env_name": f"{name}-{suite_info.name}",
"c_env_name": env_name,
"c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
"c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
})
"c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud],
"c_vm_tags": tags
}
# pylint: enable=W0640

if suite_info.owns_vm:
# create an environment for exclusive use by this suite
environment_list.append(create_environment(f"{name}-{suite_info.name}"))
else:
# add this suite to the shared environments
key: str = f"{name}-{location}"
if key in shared_environments:
shared_environments[key]["c_test_suites"].append(suite_info)
environment = shared_environments.get(key)
if environment is not None:
environment["c_test_suites"].append(suite_info)
if suite_info.template != '':
vm_tags = environment["c_vm_tags"]
if "templates" in vm_tags:
vm_tags["templates"] += ", " + suite_info.template
else:
vm_tags["templates"] = suite_info.template
else:
shared_environments[key] = {
"c_marketplace_image": marketplace_image,
"c_cloud": self.runbook.cloud,
"c_location": location,
"c_vm_size": vm_size,
"c_vhd": vhd,
"c_test_suites": [suite_info],
"c_env_name": key,
"c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
"c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
}
shared_environments[key] = create_environment(key)

environment_list.extend(shared_environments.values())

Expand All @@ -256,18 +264,17 @@ def create_environment_list(self) -> List[Dict[str, Any]]:

log: logging.Logger = logging.getLogger("lisa")
log.info("")
log.info("******** Agent Test Environments *****")
log.info("******** Waagent: Test Environments *****")
log.info("")
for environment in environment_list:
test_suites = [s.name for s in environment['c_test_suites']]
log.info("Settings for %s:\n%s\n", environment['c_env_name'], '\n'.join([f"\t{name}: {value if name != 'c_test_suites' else test_suites}" for name, value in environment.items()]))
log.info("***************************")
log.info("")

return environment_list

_URN = re.compile(r"(?P<publisher>[^\s:]+)[\s:](?P<offer>[^\s:]+)[\s:](?P<sku>[^\s:]+)[\s:](?P<version>[^\s:]+)")


@staticmethod
def _is_urn(urn: str) -> bool:
# URNs can be given as '<Publisher> <Offer> <Sku> <Version>' or '<Publisher>:<Offer>:<Sku>:<Version>'
Expand Down
Loading

0 comments on commit cb56656

Please sign in to comment.