#180 added possibility to schedule scripts

src/auth/user.py

@@ -18,3 +18,13 @@ def __str__(self) -> str:
             return self.audit_names.get(AUTH_USERNAME)
 
         return str(self.audit_names)
+
+    def as_serializable_dict(self):
+        return {
+            'user_id': self.user_id,
+            'audit_names': self.audit_names
+        }
+
+
+def from_serialized_dict(dict):
+    return User(dict['user_id'], dict['audit_names'])

src/files/user_file_storage.py

@@ -6,8 +6,8 @@
 import shutil
 import threading
 
-from utils import file_utils
-from utils.date_utils import get_current_millis, datetime_now, ms_to_datetime
+from utils import file_utils, date_utils
+from utils.date_utils import get_current_millis, ms_to_datetime
 
 LOGGER = logging.getLogger('script_server.user_file_storage')
 
@@ -55,7 +55,7 @@ def clean_results():
 
                         millis = int(timed_folder)
                         folder_date = ms_to_datetime(millis)
-                        now = datetime_now()
+                        now = date_utils.now()
 
                         if (now - folder_date) > datetime.timedelta(milliseconds=lifetime_ms):
                             folder_path = os.path.join(parent_folder, user_folder, timed_folder)

src/main.py

@@ -18,6 +18,7 @@
 from features.file_upload_feature import FileUploadFeature
 from files.user_file_storage import UserFileStorage
 from model import server_conf
+from scheduling.schedule_service import ScheduleService
 from utils import tool_utils, file_utils
 from utils.tool_utils import InvalidWebBuildException
 from web import server
@@ -122,11 +123,14 @@ def main():
     executions_callback_feature = ExecutionsCallbackFeature(execution_service, server_config.callbacks_config)
     executions_callback_feature.start()
 
+    schedule_service = ScheduleService(config_service, execution_service, CONFIG_FOLDER)
+
     server.init(
         server_config,
         server_config.authenticator,
         authorizer,
         execution_service,
+        schedule_service,
         execution_logging_service,
         config_service,
         alerts_service,

src/model/external_model.py

@@ -23,6 +23,7 @@ def config_to_external(config, id):
         'id': id,
         'name': config.name,
         'description': config.description,
+        'schedulable': config.schedulable,
         'parameters': parameters
     }
 
@@ -112,3 +113,13 @@ def server_conf_to_external(server_config, server_version):
         'enableScriptTitles': server_config.enable_script_titles,
         'version': server_version
     }
+
+
+def parse_external_schedule(external_schedule):
+    return {
+        'repeatable': external_schedule.get('repeatable'),
+        'start_datetime': external_schedule.get('startDatetime'),
+        'repeat_unit': external_schedule.get('repeatUnit'),
+        'repeat_period': external_schedule.get('repeatPeriod'),
+        'weekdays': external_schedule.get('weekDays')
+    }

src/model/model_helper.py

@@ -1,9 +1,11 @@
 import logging
 import os
 import re
+from datetime import datetime
 
 import utils.env_utils as env_utils
 from config.constants import FILE_TYPE_DIR, FILE_TYPE_FILE
+from utils import date_utils
 from utils.string_utils import is_blank
 
 ENV_VAR_PREFIX = '$$'
@@ -106,6 +108,20 @@ def read_bool_from_config(key, config_obj, *, default=None):
     raise Exception('"' + key + '" field should be true or false')
 
 
+def read_datetime_from_config(key, config_obj, *, default=None):
+    value = config_obj.get(key)
+    if value is None:
+        return default
+
+    if isinstance(value, datetime):
+        return value
+
+    if isinstance(value, str):
+        return date_utils.parse_iso_datetime(value)
+
+    raise InvalidValueTypeException('"' + key + '" field should be a datetime, but was ' + repr(value))
+
+
 def read_bool(value):
     if isinstance(value, bool):
         return value
@@ -191,7 +207,7 @@ def replace_auth_vars(text, username, audit_name):
     result = text
     if not result:
         return result
-        
+
     if not username:
         username = ''
     if not audit_name:
@@ -247,6 +263,9 @@ def __init__(self, param_name, validation_error) -> None:
         super().__init__(validation_error)
         self.param_name = param_name
 
+    def get_user_message(self):
+        return 'Invalid value for "' + self.param_name + '": ' + str(self)
+
 
 class InvalidValueTypeException(Exception):
     def __init__(self, message) -> None:

src/model/parameter_config.py

@@ -96,7 +96,7 @@ def _reload(self):
         self._reload_values()
 
     def _validate_config(self):
-        param_log_name = self._str_name()
+        param_log_name = self.str_name()
 
         if self.constant and not self.default:
             message = 'Constant should have default value specified'
@@ -106,7 +106,7 @@ def _validate_config(self):
             if not self.file_dir:
                 raise Exception('Parameter ' + param_log_name + ' has missing config file_dir')
 
-    def _str_name(self):
+    def str_name(self):
         names = (name for name in (self.name, self.param, self.description) if name)
         return next(names, 'unknown')
 

src/model/script_config.py

@@ -30,6 +30,7 @@ def __init__(self):
     'working_directory',
     'ansi_enabled',
     'output_files',
+    'schedulable',
     '_included_config')
 class ConfigModel:
 
@@ -51,6 +52,7 @@ def __init__(self,
 
         self._username = username
         self._audit_name = audit_name
+        self.schedulable = False
 
         self.parameters = ObservableList()
         self.parameter_values = ObservableDict()
@@ -63,6 +65,8 @@ def __init__(self,
 
         self._reload_config()
 
+        self.parameters.subscribe(self)
+
         self._init_parameters(username, audit_name)
 
         if parameter_values is not None:
@@ -163,6 +167,9 @@ def _reload_config(self):
 
         self.output_files = config.get('output_files', [])
 
+        if config.get('scheduling'):
+            self.schedulable = read_bool_from_config('enabled', config.get('scheduling'), default=False)
+
         if not self.script_command:
             raise Exception('No script_path is specified for ' + self.name)
 
@@ -204,6 +211,15 @@ def find_parameter(self, param_name):
                 return parameter
         return None
 
+    def on_add(self, parameter, index):
+        if self.schedulable and parameter.secure:
+            LOGGER.warning(
+                'Disabling schedulable functionality, because parameter ' + parameter.str_name() + ' is secure')
+            self.schedulable = False
+
+    def on_remove(self, parameter):
+        pass
+
     def _validate_parameter_configs(self):
         for parameter in self.parameters:
             parameter.validate_parameter_dependencies(self.parameters)

src/scheduling/schedule_config.py

@@ -0,0 +1,153 @@
+from datetime import timezone, timedelta, datetime
+
+from model import model_helper
+from utils import date_utils
+from utils.string_utils import is_blank
+
+ALLOWED_WEEKDAYS = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+
+
+def _read_start_datetime(incoming_schedule_config):
+    start_datetime = model_helper.read_datetime_from_config('start_datetime', incoming_schedule_config)
+    if start_datetime is None:
+        raise InvalidScheduleException('start_datetime is required')
+    return start_datetime
+
+
+def _read_repeat_unit(incoming_schedule_config):
+    repeat_unit = incoming_schedule_config.get('repeat_unit')
+    if is_blank(repeat_unit):
+        raise InvalidScheduleException('repeat_unit is required for repeatable schedule')
+
+    if repeat_unit.lower() not in ['hours', 'days', 'weeks', 'months']:
+        raise InvalidScheduleException('repeat_unit should be one of: hours, days, weeks, months')
+
+    return repeat_unit.lower()
+
+
+def _read_repeat_period(incoming_schedule_config):
+    period = model_helper.read_int_from_config('repeat_period', incoming_schedule_config, default=1)
+    if period <= 0:
+        raise InvalidScheduleException('repeat_period should be > 0')
+    return period
+
+
+def read_repeatable_flag(incoming_schedule_config):
+    repeatable = model_helper.read_bool_from_config('repeatable', incoming_schedule_config)
+    if repeatable is None:
+        raise InvalidScheduleException('Missing "repeatable" field')
+    return repeatable
+
+
+def read_weekdays(incoming_schedule_config):
+    weekdays = model_helper.read_list(incoming_schedule_config, 'weekdays')
+    if not weekdays:
+        raise InvalidScheduleException('At least one weekday should be specified')
+    weekdays = [day.lower().strip() for day in weekdays]
+    for day in weekdays:
+        if day not in ALLOWED_WEEKDAYS:
+            raise InvalidScheduleException('Unknown weekday: ' + day)
+    return sorted(weekdays, key=lambda x: ALLOWED_WEEKDAYS.index(x))
+
+
+def read_schedule_config(incoming_schedule_config):
+    repeatable = read_repeatable_flag(incoming_schedule_config)
+    start_datetime = _read_start_datetime(incoming_schedule_config)
+
+    prepared_schedule_config = ScheduleConfig(repeatable, start_datetime)
+    if repeatable:
+        prepared_schedule_config.repeat_unit = _read_repeat_unit(incoming_schedule_config)
+        prepared_schedule_config.repeat_period = _read_repeat_period(incoming_schedule_config)
+
+        if prepared_schedule_config.repeat_unit == 'weeks':
+            prepared_schedule_config.weekdays = read_weekdays(incoming_schedule_config)
+
+    return prepared_schedule_config
+
+
+class ScheduleConfig:
+
+    def __init__(self, repeatable, start_datetime) -> None:
+        self.repeatable = repeatable
+        self.start_datetime = start_datetime  # type: datetime
+        self.repeat_unit = None
+        self.repeat_period = None
+        self.weekdays = None
+
+    def as_serializable_dict(self):
+        result = {
+            'repeatable': self.repeatable,
+            'start_datetime': date_utils.to_iso_string(self.start_datetime)
+        }
+
+        if self.repeat_unit is not None:
+            result['repeat_unit'] = self.repeat_unit
+
+        if self.repeat_period is not None:
+            result['repeat_period'] = self.repeat_period
+
+        if self.weekdays is not None:
+            result['weekdays'] = self.weekdays
+
+        return result
+
+    def get_next_time(self):
+        if not self.repeatable:
+            return self.start_datetime
+
+        if self.repeat_unit == 'hours':
+            next_time_func = lambda start, iteration_index: start + timedelta(
+                hours=self.repeat_period * iteration_index)
+
+            get_initial_multiplier = lambda start: \
+                ((now - start).seconds // 3600 + (now - start).days * 24) \
+                // self.repeat_period
+        elif self.repeat_unit == 'days':
+            next_time_func = lambda start, iteration_index: start + timedelta(days=self.repeat_period * iteration_index)
+            get_initial_multiplier = lambda start: (now - start).days // self.repeat_period
+        elif self.repeat_unit == 'months':
+            next_time_func = lambda start, iteration_index: date_utils.add_months(start,
+                                                                                  self.repeat_period * iteration_index)
+            get_initial_multiplier = lambda start: (now - start).days // 28 // self.repeat_period
+        elif self.repeat_unit == 'weeks':
+            start_weekday = self.start_datetime.weekday()
+            offset = 0
+            for weekday in self.weekdays:
+                index = ALLOWED_WEEKDAYS.index(weekday)
+                if index < start_weekday:
+                    offset += 1
+
+            def next_weekday(start: datetime, iteration_index):
+                weeks_multiplier = (iteration_index + offset) // len(self.weekdays)
+                next_weekday_index = (iteration_index + offset) % len(self.weekdays)
+                next_weekday_name = self.weekdays[next_weekday_index]
+                next_weekday = ALLOWED_WEEKDAYS.index(next_weekday_name)
+
+                return start \
+                       + timedelta(weeks=self.repeat_period * weeks_multiplier) \
+                       + timedelta(days=(next_weekday - start.weekday()))
+
+            next_time_func = next_weekday
+
+            get_initial_multiplier = lambda start: (now - start).days // 7 // self.repeat_period * len(
+                self.weekdays) - 1
+        else:
+            raise Exception('Unknown unit: ' + repr(self.repeat_unit))
+
+        now = date_utils.now(tz=timezone.utc)
+        max_iterations = 10000
+        initial_multiplier = max(0, get_initial_multiplier(self.start_datetime))
+        i = 0
+        while True:
+            resolved_time = next_time_func(self.start_datetime, i + initial_multiplier)
+            if resolved_time >= now:
+                return resolved_time
+
+            i += 1
+            if i > max_iterations:
+                raise Exception('Endless loop in calc next time')
+
+
+class InvalidScheduleException(Exception):
+    def __init__(self, message) -> None:
+        super().__init__(message)