Requirements database [v2]

avocado/core/plugin_interfaces.py

@@ -364,6 +364,46 @@ async def check_task_requirements(runtime_task):
         :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
         """
 
+    @staticmethod
+    @abc.abstractmethod
+    async def is_requirement_in_cache(runtime_task):
+        """Checks if it's necessary to run the requirement.
+
+        There are occasions when the similar requirement has been run and its
+        results are already saved in cache. In such occasion, it is not
+        necessary to run the task again. For example, this might be useful for
+        tasks which would install the same package to the same environment.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        :return: If the results are already in cache.
+        :rtype: True if task is in cache
+                False if task is not in cache
+                None if task is running in different process and should be in
+                cache soon.
+        """
+
+    @staticmethod
+    @abc.abstractmethod
+    async def save_requirement_in_cache(runtime_task):
+        """Saves the information about requirement in cache before
+        the runtime_task is run.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        """
+
+    @staticmethod
+    @abc.abstractmethod
+    async def update_requirement_cache(runtime_task, result):
+        """Updates the information about requirement in cache based on result.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        :param result: result of runtime_task
+        :type result: `avocado.core.teststatus.STATUSES`
+        """
+
 
 class DeploymentSpawner(Spawner):
     """Spawners that needs basic deployment are based on this class.

avocado/core/spawners/mock.py

@@ -47,6 +47,18 @@ async def terminate_task(self, runtime_task):
     async def check_task_requirements(runtime_task):
         return True
 
+    @staticmethod
+    async def is_requirement_in_cache(runtime_task):
+        return False
+
+    @staticmethod
+    async def save_requirement_in_cache(runtime_task):
+        pass
+
+    @staticmethod
+    async def update_requirement_cache(runtime_task, result):
+        pass
+
 
 class MockRandomAliveSpawner(MockSpawner):
     """A mocking spawner that simulates randomness about tasks being alive."""

avocado/core/task/statemachine.py

@@ -21,6 +21,7 @@ def __init__(self, tasks, status_repo):
         self._started = []
         self._finished = []
         self._lock = asyncio.Lock()
+        self._cache_lock = asyncio.Lock()
 
     @property
     def requested(self):
@@ -46,6 +47,10 @@ def finished(self):
     def lock(self):
         return self._lock
 
+    @property
+    def cache_lock(self):
+        return self._cache_lock
+
     @property
     async def complete(self):
         async with self._lock:
@@ -188,6 +193,24 @@ async def triage(self):
                 LOG.debug('Task "%s" has failed dependencies',
                           runtime_task.task.identifier)
                 return
+        if runtime_task.task.category != "test":
+            async with self._state_machine.cache_lock:
+                is_task_in_cache = await self._spawner.is_requirement_in_cache(
+                    runtime_task)
+                if is_task_in_cache is None:
+                    async with self._state_machine.lock:
+                        self._state_machine.triaging.append(runtime_task)
+                        runtime_task.status = 'WAITING'
+                        await asyncio.sleep(0.1)
+                    return
+
+                if is_task_in_cache:
+                    await self._state_machine.finish_task(
+                        runtime_task, "FINISHED: Task in cache")
+                    runtime_task.result = 'pass'
+                    return
+
+                await self._spawner.save_requirement_in_cache(runtime_task)
 
         # the task is ready to run
         async with self._state_machine.lock:
@@ -267,6 +290,10 @@ async def monitor(self):
             latest_task_data = \
                 self._state_machine._status_repo.get_latest_task_data(
                     str(runtime_task.task.identifier)) or {}
+        if runtime_task.task.category != "test":
+            async with self._state_machine.cache_lock:
+                await self._spawner.update_requirement_cache(
+                    runtime_task, latest_task_data['result'].upper())
         runtime_task.result = latest_task_data['result']
         result_stats = set(key.upper()for key in
                            self._state_machine._status_repo.result_stats.keys())

avocado/plugins/spawners/podman.py

@@ -3,10 +3,13 @@
 import logging
 import os
 import subprocess
+import uuid
 
+from avocado.core.dependencies.requirements import cache
 from avocado.core.plugin_interfaces import CLI, DeploymentSpawner, Init
 from avocado.core.settings import settings
 from avocado.core.spawners.common import SpawnerMixin, SpawnMethod
+from avocado.core.teststatus import STATUSES_NOT_OK
 from avocado.core.version import VERSION
 from avocado.utils import distro
 from avocado.utils.asset import Asset
@@ -96,6 +99,10 @@ class PodmanSpawner(DeploymentSpawner, SpawnerMixin):
 
     _PYTHON_VERSIONS_CACHE = {}
 
+    def __init__(self, config=None, job=None):
+        SpawnerMixin.__init__(self, config, job)
+        self.environment = f"podman:{self.config.get('spawner.podman.image')}"
+
     def is_task_alive(self, runtime_task):
         if runtime_task.spawner_handle is None:
             return False
@@ -211,7 +218,9 @@ async def _create_container_for_task(self, runtime_task, env_args,
                            (f"{test_output}:"
                             f"{os.path.expanduser(podman_output)}"))
 
-        image = self.config.get('spawner.podman.image')
+        image, _ = self._get_image_from_cache(runtime_task)
+        if not image:
+            image = self.config.get('spawner.podman.image')
 
         envs = [f"-e={k}={v}" for k, v in env_args.items()]
         try:
@@ -294,3 +303,78 @@ async def check_task_requirements(runtime_task):
         if runtime_task.task.runnable.runner_command() is None:
             return False
         return True
+
+    async def update_requirement_cache(self, runtime_task, result):
+        environment_id, _ = self._get_image_from_cache(runtime_task, True)
+        if result in STATUSES_NOT_OK:
+            cache.delete_environment(self.environment, environment_id)
+            return
+        _, stdout, _ = await self.podman.execute("commit", "-q",
+                                                 runtime_task.spawner_handle)
+        container_id = stdout.decode().strip()
+        cache.update_environment(self.environment,
+                                 environment_id,
+                                 container_id)
+        cache.update_requirement_status(self.environment,
+                                        container_id,
+                                        runtime_task.task.runnable.kind,
+                                        runtime_task.task.runnable.kwargs.get(
+                                            'name'),
+                                        True)
+
+    async def save_requirement_in_cache(self, runtime_task):
+        container_id = str(uuid.uuid4())
+        _, requirements = self._get_image_from_cache(runtime_task)
+        if requirements:
+            for requirement_type, requirement in requirements:
+                cache.set_requirement(self.environment, container_id,
+                                      requirement_type, requirement)
+        cache.set_requirement(self.environment,
+                              container_id,
+                              runtime_task.task.runnable.kind,
+                              runtime_task.task.runnable.kwargs.get('name'),
+                              False)
+
+    async def is_requirement_in_cache(self, runtime_task):
+        environment, _ = self._get_image_from_cache(runtime_task,
+                                                    use_task=True)
+        if not environment:
+            return False
+        if cache.is_environment_prepared(environment):
+            return True
+        return None
+
+    def _get_image_from_cache(self, runtime_task, use_task=False):
+
+        def _get_all_finished_requirements(requirement_tasks):
+            all_finished_requirements = []
+            for requirement in requirement_tasks:
+                all_finished_requirements.extend(_get_all_finished_requirements(
+                    requirement.dependencies))
+                runnable = requirement.task.runnable
+                all_finished_requirements.append((runnable.kind,
+                                                  runnable.kwargs.get('name')))
+            return all_finished_requirements
+
+        finished_requirements = []
+        if use_task:
+            finished_requirements.append(
+                (runtime_task.task.runnable.kind,
+                 runtime_task.task.runnable.kwargs.get('name')))
+        finished_requirements.extend(
+            _get_all_finished_requirements(runtime_task.dependencies))
+        if not finished_requirements:
+            return None, None
+
+        runtime_task_kind, runtime_task_name = finished_requirements[0]
+        cache_entries = cache.get_all_environments_with_requirement(
+            self.environment,
+            runtime_task_kind,
+            runtime_task_name)
+        if not cache_entries:
+            return None, None
+        for image, requirements in cache_entries.items():
+            if len(finished_requirements) == len(requirements):
+                if set(requirements) == set(finished_requirements):
+                    return image, requirements
+        return None, None

avocado/plugins/spawners/process.py

@@ -1,9 +1,15 @@
 import asyncio
 import os
+import socket
 import sys
 
+from avocado.core.dependencies.requirements import cache
 from avocado.core.plugin_interfaces import Spawner
 from avocado.core.spawners.common import SpawnerMixin, SpawnMethod
+from avocado.core.teststatus import STATUSES_NOT_OK
+
+ENVIRONMENT_TYPE = 'local'
+ENVIRONMENT = socket.gethostname()
 
 
 class ProcessSpawner(Spawner, SpawnerMixin):
@@ -64,3 +70,33 @@ async def check_task_requirements(runtime_task):
         if runtime_task.task.runnable.runner_command() is None:
             return False
         return True
+
+    @staticmethod
+    async def update_requirement_cache(runtime_task, result):
+        kind = runtime_task.task.runnable.kind
+        name = runtime_task.task.runnable.kwargs.get('name')
+        cache.set_requirement(ENVIRONMENT_TYPE, ENVIRONMENT, kind, name)
+        if result in STATUSES_NOT_OK:
+            cache.delete_requirement(ENVIRONMENT_TYPE, ENVIRONMENT, kind, name)
+            return
+        cache.update_requirement_status(ENVIRONMENT_TYPE,
+                                        ENVIRONMENT,
+                                        kind,
+                                        name,
+                                        True)
+
+    @staticmethod
+    async def is_requirement_in_cache(runtime_task):
+        kind = runtime_task.task.runnable.kind
+        name = runtime_task.task.runnable.kwargs.get('name')
+        return cache.is_requirement_in_cache(ENVIRONMENT_TYPE,
+                                             ENVIRONMENT,
+                                             kind,
+                                             name)
+
+    @staticmethod
+    async def save_requirement_in_cache(runtime_task):
+        kind = runtime_task.task.runnable.kind
+        name = runtime_task.task.runnable.kwargs.get('name')
+        cache.set_requirement(ENVIRONMENT_TYPE, ENVIRONMENT, kind, name,
+                              False)

docs/source/guides/user/chapters/requirements.rst

@@ -19,6 +19,21 @@ it is started.
 
 When any of the requirements defined on a test fails, the test is skipped.
 
+When the requirement is fulfilled, it will be saved into the avocado cache, and
+it will be reused by other tests.
+
+Also, the requirement will stay in cache after the Avocado run, so the second
+run of the tests will use requirements from cache, which will make tests more
+efficient.
+
+.. warning::
+
+        If any environment is modified without Avocado knowing about it 
+        (packages being uninstalled, podman images removed, etc), the 
+        requirement resolution behavior is undefined and will probably crash. 
+        If such a change is made to the environment, it's recommended to clear 
+        the requirements cache file.
+
 Defining a test requirement
 ---------------------------
 

selftests/check.py

@@ -203,13 +203,14 @@ def parse_args():
         epilog='''\
 The list of test availables for --skip and --select are:
 
-  static-checks      Run static checks (isort, lint, etc)
-  job-api            Run job API checks
-  nrunner-interface  Run selftests/functional/test_nrunner_interface.py
-  unit               Run selftests/unit/
-  jobs               Run selftests/jobs/
-  functional         Run selftests/functional/
-  optional-plugins   Run optional_plugins/*/tests/
+  static-checks         Run static checks (isort, lint, etc)
+  job-api               Run job API checks
+  nrunner-interface     Run selftests/functional/test_nrunner_interface.py
+  nrunner-requirement   Run selftests/functional/serial/test_requirements.py
+  unit                  Run selftests/unit/
+  jobs                  Run selftests/jobs/
+  functional            Run selftests/functional/
+  optional-plugins      Run optional_plugins/*/tests/
         ''')
     group = parser.add_mutually_exclusive_group()
     parser.add_argument('-f',
@@ -568,6 +569,22 @@ def create_suites(args):  # pylint: disable=W0621
     if args.dict_tests['nrunner-interface']:
         suites.append(TestSuite.from_config(config_nrunner_interface, "nrunner-interface"))
 
+    # ========================================================================
+    # Run functional requirement tests
+    # ========================================================================
+    config_nrunner_requirement = {
+        'resolver.references': ['selftests/functional/serial/test_requirements.py'],
+        'nrunner.max_parallel_tasks': 1,
+        'run.dict_variants': [
+            {'spawner': 'process'},
+
+            {'spawner': 'podman'},
+        ]
+    }
+
+    if args.dict_tests['nrunner-requirement']:
+        suites.append(TestSuite.from_config(config_nrunner_requirement, "nrunner-requirement"))
+
     # ========================================================================
     # Run all static checks, unit and functional tests
     # ========================================================================
@@ -629,6 +646,7 @@ def main(args):  # pylint: disable=W0621
         'static-checks': False,
         'job-api': False,
         'nrunner-interface': False,
+        'nrunner-requirement': False,
         'unit': False,
         'jobs': False,
         'functional': False,