Requirements database [v2]

avocado/core/dependencies/requirements/cache/__init__.py

@@ -1,3 +1,6 @@
 # The sqlite based backend is the only implementation
 from avocado.core.dependencies.requirements.cache.backends.sqlite import (
-    get_requirement, set_requirement)
+    delete_environment, delete_requirement,
+    get_all_environments_with_requirement, is_environment_prepared,
+    is_requirement_in_cache, set_requirement, update_environment,
+    update_requirement_status)

avocado/core/dependencies/requirements/cache/backends/sqlite.py

@@ -24,6 +24,9 @@
 #: The location of the requirements cache database
 CACHE_DATABASE_PATH = get_datafile_path('cache', 'requirements.sqlite')
 
+sqlite3.register_adapter(bool, int)
+sqlite3.register_converter("BOOLEAN", lambda v: bool(int(v)))
+
 #: The definition of the database schema
 SCHEMA = [
     'CREATE TABLE IF NOT EXISTS requirement_type (requirement_type TEXT UNIQUE)',
@@ -41,6 +44,7 @@
      'environment TEXT,'
      'requirement_type TEXT,'
      'requirement TEXT,'
+     'saved BOOLEAN,'
      'FOREIGN KEY(environment_type) REFERENCES environment(environment_type),'
      'FOREIGN KEY(environment) REFERENCES environment(environment),'
      'FOREIGN KEY(requirement_type) REFERENCES requirement_type(requirement_type)'
@@ -51,6 +55,7 @@
 
 
 def _create_requirement_cache_db():
+    os.makedirs(os.path.dirname(CACHE_DATABASE_PATH), exist_ok=True)
     with sqlite3.connect(CACHE_DATABASE_PATH) as conn:
         cursor = conn.cursor()
         for entry in SCHEMA:
@@ -59,7 +64,7 @@ def _create_requirement_cache_db():
 
 
 def set_requirement(environment_type, environment,
-                    requirement_type, requirement):
+                    requirement_type, requirement, saved=True):
     if not os.path.exists(CACHE_DATABASE_PATH):
         _create_requirement_cache_db()
 
@@ -71,18 +76,24 @@ def set_requirement(environment_type, environment,
         cursor.execute(sql, (environment_type, environment))
         sql = "INSERT OR IGNORE INTO requirement_type VALUES (?)"
         cursor.execute(sql, (requirement_type, ))
-        sql = "INSERT OR IGNORE INTO requirement VALUES (?, ?, ?, ?)"
+        sql = "INSERT OR IGNORE INTO requirement VALUES (?, ?, ?, ?, ?)"
         cursor.execute(sql, (environment_type, environment,
-                             requirement_type, requirement))
-    conn.commit()
+                             requirement_type, requirement, saved))
+        conn.commit()
 
 
-def get_requirement(environment_type, environment,
-                    requirement_type, requirement):
+def is_requirement_in_cache(environment_type, environment,
+                            requirement_type, requirement):
+    """Checks if requirement is in cache.
+
+       :rtype: True if requirement is in cache
+               False if requirement is not in cache
+               None if requirement is in cache but it is not saved yet.
+    """
     if not os.path.exists(CACHE_DATABASE_PATH):
         return False
 
-    sql = ("SELECT COUNT(*) FROM requirement WHERE ("
+    sql = ("SELECT r.saved FROM requirement r WHERE ("
            "environment_type = ? AND "
            "environment = ? AND "
            "requirement_type = ? AND "
@@ -94,5 +105,196 @@ def get_requirement(environment_type, environment,
                                       requirement_type, requirement))
         row = result.fetchone()
         if row is not None:
-            return row[0] == 1
+            if row[0]:
+                return True
+            return None
+    return False
+
+
+def is_environment_prepared(environment):
+    """Checks if environment has all requirements saved."""
+
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return False
+
+    sql = ("SELECT COUNT(*) FROM requirement r JOIN "
+           "environment e ON e.environment = r.environment "
+           "WHERE (r.environment = ? AND "
+           "r.saved = 0)")
+
+    with sqlite3.connect(CACHE_DATABASE_PATH,
+                         detect_types=sqlite3.PARSE_DECLTYPES) as conn:
+        cursor = conn.cursor()
+        result = cursor.execute(sql, (environment,))
+
+        row = result.fetchone()
+        if row is not None:
+            return row[0] == 0
     return False
+
+
+def update_environment(environment_type, old_environment, new_environment):
+    """Updates environment information for each requirement in one environment.
+
+    It will remove the old environment and add the new one to the cache.
+
+    :param environment_type: Type of fetched environment
+    :type environment_type: str
+    :param old_environment: Environment which should be updated
+    :type environment: str
+    :param new_environment: Environment, which will be a reimbursement for the
+                            old one.
+    :type environment: str
+    """
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return False
+
+    with sqlite3.connect(CACHE_DATABASE_PATH) as conn:
+        cursor = conn.cursor()
+        sql = "INSERT OR IGNORE INTO environment VALUES (?, ?)"
+        cursor.execute(sql, (environment_type, new_environment))
+
+        sql = ("UPDATE requirement SET environment = ? WHERE ("
+               "environment_type = ? AND "
+               "environment = ? )")
+
+        cursor.execute(sql, (new_environment, environment_type,
+                             old_environment))
+
+        sql = ("DELETE FROM environment WHERE ("
+               "environment_type = ? AND "
+               "environment = ? )")
+
+        cursor.execute(sql, (environment_type, old_environment))
+        conn.commit()
+
+
+def update_requirement_status(environment_type, environment, requirement_type,
+                              requirement, new_status):
+    """Updates status of selected requirement in cache.
+
+    The status has two values, save=True or not_save=False.
+
+    :param environment_type: Type of fetched environment
+    :type environment_type: str
+    :param environment: Environment where the requirement is
+    :type environment: str
+    :param requirement_type: Type of the requirement in environment
+    :type requirement_type: str
+    :param requirement: Name of requirement
+    :type requirement: str
+    :param new_status: Requirement status which will be updated
+    :type new_status: bool
+    """
+
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return False
+
+    sql = ("UPDATE requirement SET saved = ? WHERE ("
+           "environment_type = ? AND "
+           "environment = ? AND "
+           "requirement_type = ? AND "
+           "requirement = ?)")
+
+    with sqlite3.connect(CACHE_DATABASE_PATH) as conn:
+        cursor = conn.cursor()
+        cursor.execute(sql, (new_status, environment_type, environment,
+                             requirement_type, requirement))
+        conn.commit()
+
+    return True
+
+
+def delete_environment(environment_type, environment):
+    """Deletes environment with all its requirements from cache.
+
+    :param environment_type: Type of environment
+    :type environment_type: str
+    :param environment: Environment which will be deleted
+    :type environment: str
+    """
+
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return False
+
+    with sqlite3.connect(CACHE_DATABASE_PATH) as conn:
+        sql = ("DELETE FROM requirement WHERE ("
+               "environment_type = ? AND "
+               "environment = ? )")
+        cursor = conn.cursor()
+        cursor.execute(sql, (environment_type, environment))
+        sql = ("DELETE FROM environment WHERE ("
+               "environment_type = ? AND "
+               "environment = ? )")
+        cursor.execute(sql, (environment_type, environment))
+        conn.commit()
+
+
+def delete_requirement(environment_type, environment, requirement_type,
+                       requirement):
+    """Deletes requirement from cache.
+
+    :param environment_type: Type of environment
+    :type environment_type: str
+    :param environment: Environment where the requirement is.
+    :type environment: str
+    :param requirement_type: Type of the requirement in environment
+    :type requirement_type: str
+    :param requirement: Name of requirement which will be deleted
+    :type requirement: str
+    """
+
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return False
+
+    with sqlite3.connect(CACHE_DATABASE_PATH) as conn:
+        sql = ("DELETE FROM requirement WHERE ("
+               "environment_type = ? AND "
+               "environment = ? AND "
+               "requirement_type = ? AND "
+               "requirement = ?)")
+        cursor = conn.cursor()
+        cursor.execute(sql, (environment_type, environment, requirement_type,
+                             requirement))
+        conn.commit()
+
+
+def get_all_environments_with_requirement(environment_type, requirement_type,
+                                          requirement):
+    """Fetches all environments with selected requirement from cache.
+
+    :param environment_type: Type of fetched environment
+    :type environment_type: str
+    :param requirement_type: Type of the requirement in environment
+    :type requirement_type: str
+    :param requirement: Name of requirement
+    :type requirement: str
+    :return: Dict with all environments which has selected requirements.
+
+    """
+    requirements = {}
+    if not os.path.exists(CACHE_DATABASE_PATH):
+        return requirements
+
+    environment_select = ("SELECT e.environment FROM requirement r JOIN "
+                          "environment e ON e.environment = r.environment "
+                          "WHERE (r.environment_type = ? AND "
+                          "r.requirement_type = ? AND "
+                          "r.requirement = ?)")
+    sql = (f"SELECT r.environment, r.requirement_type, r.requirement "
+           f"FROM requirement AS r, ({environment_select}) AS e "
+           f"WHERE r.environment = e.environment")
+
+    with sqlite3.connect(CACHE_DATABASE_PATH,
+                         detect_types=sqlite3.PARSE_DECLTYPES) as conn:
+        cursor = conn.cursor()
+        result = cursor.execute(sql, (environment_type,
+                                      requirement_type,
+                                      requirement))
+
+        for row in result.fetchall():
+            if row[0] in requirements:
+                requirements[row[0]].append((row[1], row[2]))
+            else:
+                requirements[row[0]] = [(row[1], row[2])]
+    return requirements

avocado/core/plugin_interfaces.py

@@ -364,6 +364,46 @@ async def check_task_requirements(runtime_task):
         :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
         """
 
+    @staticmethod
+    @abc.abstractmethod
+    async def is_requirement_in_cache(runtime_task):
+        """Checks if it's necessary to run the requirement.
+
+        There are occasions when the similar requirement has been run and its
+        results are already saved in cache. In such occasion, it is not
+        necessary to run the task again. For example, this might be useful for
+        tasks which would install the same package to the same environment.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        :return: If the results are already in cache.
+        :rtype: True if task is in cache
+                False if task is not in cache
+                None if task is running in different process and should be in
+                cache soon.
+        """
+
+    @staticmethod
+    @abc.abstractmethod
+    async def save_requirement_in_cache(runtime_task):
+        """Saves the information about requirement in cache before
+        the runtime_task is run.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        """
+
+    @staticmethod
+    @abc.abstractmethod
+    async def update_requirement_cache(runtime_task, result):
+        """Updates the information about requirement in cache based on result.
+
+        :param runtime_task: runtime task with requirement
+        :type runtime_task: :class:`avocado.core.task.runtime.RuntimeTask`
+        :param result: result of runtime_task
+        :type result: `avocado.core.teststatus.STATUSES`
+        """
+
 
 class DeploymentSpawner(Spawner):
     """Spawners that needs basic deployment are based on this class.

avocado/core/spawners/mock.py

@@ -47,6 +47,18 @@ async def terminate_task(self, runtime_task):
     async def check_task_requirements(runtime_task):
         return True
 
+    @staticmethod
+    async def is_requirement_in_cache(runtime_task):
+        return False
+
+    @staticmethod
+    async def save_requirement_in_cache(runtime_task):
+        pass
+
+    @staticmethod
+    async def update_requirement_cache(runtime_task, result):
+        pass
+
 
 class MockRandomAliveSpawner(MockSpawner):
     """A mocking spawner that simulates randomness about tasks being alive."""