From 519d2172abe4a2fd49888e98e48a905f8a70a58f Mon Sep 17 00:00:00 2001 From: yshepilov Date: Sun, 26 Jul 2020 17:42:49 +0200 Subject: [PATCH] #180 added possibility to schedule scripts --- src/auth/user.py | 10 + src/files/user_file_storage.py | 6 +- src/main.py | 4 + src/model/external_model.py | 11 + src/model/model_helper.py | 21 +- src/model/parameter_config.py | 4 +- src/model/script_config.py | 16 + src/scheduling/__init__.py | 0 src/scheduling/schedule_config.py | 153 ++++++++ src/scheduling/schedule_service.py | 177 +++++++++ src/scheduling/scheduling_job.py | 35 ++ src/tests/__init__.py | 0 src/tests/date_utils_test.py | 167 +++++++- src/tests/external_model_test.py | 43 +- src/tests/file_utils_test.py | 20 + src/tests/model_helper_test.py | 37 ++ src/tests/scheduling/__init__.py | 0 src/tests/scheduling/schedule_config_test.py | 128 ++++++ src/tests/scheduling/schedule_service_test.py | 369 +++++++++++++++++ src/tests/scheduling/scheduling_job_test.py | 36 ++ src/tests/script_config_test.py | 41 +- src/tests/test_utils.py | 15 +- src/tests/web/server_test.py | 1 + src/utils/date_utils.py | 43 +- src/utils/file_utils.py | 4 +- src/web/server.py | 45 ++- .../materializecss/material-datepicker.css | 3 + .../css/materializecss/material-textfield.css | 15 + .../common/components/PromisableButton.vue | 43 +- web-src/src/common/components/combobox.vue | 12 +- .../common/components/inputs/DatePicker.vue | 76 ++++ .../common/components/inputs/TimePicker.vue | 96 +++++ web-src/src/common/components/textfield.vue | 1 + .../materializecss/color_variables.scss | 31 -- .../materializecss/imports/datepicker.js | 8 + .../common/materializecss/imports/modal.js | 1 + web-src/src/common/style_imports.js | 5 +- .../components/schedule/SchedulePanel.vue | 370 ++++++++++++++++++ .../components/schedule/ToggleDayButton.vue | 48 +++ .../schedule/schedulePanelFields.js | 11 + .../components/scripts/ScheduleButton.vue | 74 ++++ .../scripts/ScriptViewScheduleHolder.vue | 121 ++++++ .../scripts/script-parameters-view.vue | 16 - .../components/scripts/script-view.vue | 117 +++++- web-src/src/main-app/store/index.js | 4 +- web-src/src/main-app/store/mainStoreHelper.js | 18 + .../main-app/store/scriptExecutionManager.js | 14 +- web-src/src/main-app/store/scriptSchedule.js | 28 ++ web-src/tests/unit/combobox_test.js | 20 +- .../components/inputs/TimePicker_test.js | 138 +++++++ web-src/vue.config.js | 1 - 51 files changed, 2535 insertions(+), 122 deletions(-) create mode 100644 src/scheduling/__init__.py create mode 100644 src/scheduling/schedule_config.py create mode 100644 src/scheduling/schedule_service.py create mode 100644 src/scheduling/scheduling_job.py mode change 100755 => 100644 src/tests/__init__.py create mode 100644 src/tests/file_utils_test.py create mode 100755 src/tests/scheduling/__init__.py create mode 100644 src/tests/scheduling/schedule_config_test.py create mode 100644 src/tests/scheduling/schedule_service_test.py create mode 100644 src/tests/scheduling/scheduling_job_test.py create mode 100644 web-src/src/assets/css/materializecss/material-datepicker.css create mode 100644 web-src/src/assets/css/materializecss/material-textfield.css create mode 100644 web-src/src/common/components/inputs/DatePicker.vue create mode 100644 web-src/src/common/components/inputs/TimePicker.vue delete mode 100644 web-src/src/common/materializecss/color_variables.scss create mode 100644 web-src/src/common/materializecss/imports/datepicker.js create mode 100644 web-src/src/main-app/components/schedule/SchedulePanel.vue create mode 100644 web-src/src/main-app/components/schedule/ToggleDayButton.vue create mode 100644 web-src/src/main-app/components/schedule/schedulePanelFields.js create mode 100644 web-src/src/main-app/components/scripts/ScheduleButton.vue create mode 100644 web-src/src/main-app/components/scripts/ScriptViewScheduleHolder.vue create mode 100644 web-src/src/main-app/store/mainStoreHelper.js create mode 100644 web-src/src/main-app/store/scriptSchedule.js create mode 100644 web-src/tests/unit/common/components/inputs/TimePicker_test.js diff --git a/src/auth/user.py b/src/auth/user.py index d8889c8d..51242c4e 100644 --- a/src/auth/user.py +++ b/src/auth/user.py @@ -18,3 +18,13 @@ def __str__(self) -> str: return self.audit_names.get(AUTH_USERNAME) return str(self.audit_names) + + def as_serializable_dict(self): + return { + 'user_id': self.user_id, + 'audit_names': self.audit_names + } + + +def from_serialized_dict(dict): + return User(dict['user_id'], dict['audit_names']) diff --git a/src/files/user_file_storage.py b/src/files/user_file_storage.py index 6a533b99..4acaf78f 100644 --- a/src/files/user_file_storage.py +++ b/src/files/user_file_storage.py @@ -6,8 +6,8 @@ import shutil import threading -from utils import file_utils -from utils.date_utils import get_current_millis, datetime_now, ms_to_datetime +from utils import file_utils, date_utils +from utils.date_utils import get_current_millis, ms_to_datetime LOGGER = logging.getLogger('script_server.user_file_storage') @@ -55,7 +55,7 @@ def clean_results(): millis = int(timed_folder) folder_date = ms_to_datetime(millis) - now = datetime_now() + now = date_utils.now() if (now - folder_date) > datetime.timedelta(milliseconds=lifetime_ms): folder_path = os.path.join(parent_folder, user_folder, timed_folder) diff --git a/src/main.py b/src/main.py index ada497db..714dc943 100644 --- a/src/main.py +++ b/src/main.py @@ -18,6 +18,7 @@ from features.file_upload_feature import FileUploadFeature from files.user_file_storage import UserFileStorage from model import server_conf +from scheduling.schedule_service import ScheduleService from utils import tool_utils, file_utils from utils.tool_utils import InvalidWebBuildException from web import server @@ -122,11 +123,14 @@ def main(): executions_callback_feature = ExecutionsCallbackFeature(execution_service, server_config.callbacks_config) executions_callback_feature.start() + schedule_service = ScheduleService(config_service, execution_service, CONFIG_FOLDER) + server.init( server_config, server_config.authenticator, authorizer, execution_service, + schedule_service, execution_logging_service, config_service, alerts_service, diff --git a/src/model/external_model.py b/src/model/external_model.py index daf32a7d..2287e100 100644 --- a/src/model/external_model.py +++ b/src/model/external_model.py @@ -23,6 +23,7 @@ def config_to_external(config, id): 'id': id, 'name': config.name, 'description': config.description, + 'schedulable': config.schedulable, 'parameters': parameters } @@ -112,3 +113,13 @@ def server_conf_to_external(server_config, server_version): 'enableScriptTitles': server_config.enable_script_titles, 'version': server_version } + + +def parse_external_schedule(external_schedule): + return { + 'repeatable': external_schedule.get('repeatable'), + 'start_datetime': external_schedule.get('startDatetime'), + 'repeat_unit': external_schedule.get('repeatUnit'), + 'repeat_period': external_schedule.get('repeatPeriod'), + 'weekdays': external_schedule.get('weekDays') + } diff --git a/src/model/model_helper.py b/src/model/model_helper.py index 61fc5091..a12941cd 100644 --- a/src/model/model_helper.py +++ b/src/model/model_helper.py @@ -1,9 +1,11 @@ import logging import os import re +from datetime import datetime import utils.env_utils as env_utils from config.constants import FILE_TYPE_DIR, FILE_TYPE_FILE +from utils import date_utils from utils.string_utils import is_blank ENV_VAR_PREFIX = '$$' @@ -106,6 +108,20 @@ def read_bool_from_config(key, config_obj, *, default=None): raise Exception('"' + key + '" field should be true or false') +def read_datetime_from_config(key, config_obj, *, default=None): + value = config_obj.get(key) + if value is None: + return default + + if isinstance(value, datetime): + return value + + if isinstance(value, str): + return date_utils.parse_iso_datetime(value) + + raise InvalidValueTypeException('"' + key + '" field should be a datetime, but was ' + repr(value)) + + def read_bool(value): if isinstance(value, bool): return value @@ -191,7 +207,7 @@ def replace_auth_vars(text, username, audit_name): result = text if not result: return result - + if not username: username = '' if not audit_name: @@ -247,6 +263,9 @@ def __init__(self, param_name, validation_error) -> None: super().__init__(validation_error) self.param_name = param_name + def get_user_message(self): + return 'Invalid value for "' + self.param_name + '": ' + str(self) + class InvalidValueTypeException(Exception): def __init__(self, message) -> None: diff --git a/src/model/parameter_config.py b/src/model/parameter_config.py index c267e937..34b0525e 100644 --- a/src/model/parameter_config.py +++ b/src/model/parameter_config.py @@ -96,7 +96,7 @@ def _reload(self): self._reload_values() def _validate_config(self): - param_log_name = self._str_name() + param_log_name = self.str_name() if self.constant and not self.default: message = 'Constant should have default value specified' @@ -106,7 +106,7 @@ def _validate_config(self): if not self.file_dir: raise Exception('Parameter ' + param_log_name + ' has missing config file_dir') - def _str_name(self): + def str_name(self): names = (name for name in (self.name, self.param, self.description) if name) return next(names, 'unknown') diff --git a/src/model/script_config.py b/src/model/script_config.py index 1257c834..dfeb3b25 100644 --- a/src/model/script_config.py +++ b/src/model/script_config.py @@ -30,6 +30,7 @@ def __init__(self): 'working_directory', 'ansi_enabled', 'output_files', + 'schedulable', '_included_config') class ConfigModel: @@ -51,6 +52,7 @@ def __init__(self, self._username = username self._audit_name = audit_name + self.schedulable = False self.parameters = ObservableList() self.parameter_values = ObservableDict() @@ -63,6 +65,8 @@ def __init__(self, self._reload_config() + self.parameters.subscribe(self) + self._init_parameters(username, audit_name) if parameter_values is not None: @@ -163,6 +167,9 @@ def _reload_config(self): self.output_files = config.get('output_files', []) + if config.get('scheduling'): + self.schedulable = read_bool_from_config('enabled', config.get('scheduling'), default=False) + if not self.script_command: raise Exception('No script_path is specified for ' + self.name) @@ -204,6 +211,15 @@ def find_parameter(self, param_name): return parameter return None + def on_add(self, parameter, index): + if self.schedulable and parameter.secure: + LOGGER.warning( + 'Disabling schedulable functionality, because parameter ' + parameter.str_name() + ' is secure') + self.schedulable = False + + def on_remove(self, parameter): + pass + def _validate_parameter_configs(self): for parameter in self.parameters: parameter.validate_parameter_dependencies(self.parameters) diff --git a/src/scheduling/__init__.py b/src/scheduling/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scheduling/schedule_config.py b/src/scheduling/schedule_config.py new file mode 100644 index 00000000..35c3facb --- /dev/null +++ b/src/scheduling/schedule_config.py @@ -0,0 +1,153 @@ +from datetime import timezone, timedelta, datetime + +from model import model_helper +from utils import date_utils +from utils.string_utils import is_blank + +ALLOWED_WEEKDAYS = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + + +def _read_start_datetime(incoming_schedule_config): + start_datetime = model_helper.read_datetime_from_config('start_datetime', incoming_schedule_config) + if start_datetime is None: + raise InvalidScheduleException('start_datetime is required') + return start_datetime + + +def _read_repeat_unit(incoming_schedule_config): + repeat_unit = incoming_schedule_config.get('repeat_unit') + if is_blank(repeat_unit): + raise InvalidScheduleException('repeat_unit is required for repeatable schedule') + + if repeat_unit.lower() not in ['hours', 'days', 'weeks', 'months']: + raise InvalidScheduleException('repeat_unit should be one of: hours, days, weeks, months') + + return repeat_unit.lower() + + +def _read_repeat_period(incoming_schedule_config): + period = model_helper.read_int_from_config('repeat_period', incoming_schedule_config, default=1) + if period <= 0: + raise InvalidScheduleException('repeat_period should be > 0') + return period + + +def read_repeatable_flag(incoming_schedule_config): + repeatable = model_helper.read_bool_from_config('repeatable', incoming_schedule_config) + if repeatable is None: + raise InvalidScheduleException('Missing "repeatable" field') + return repeatable + + +def read_weekdays(incoming_schedule_config): + weekdays = model_helper.read_list(incoming_schedule_config, 'weekdays') + if not weekdays: + raise InvalidScheduleException('At least one weekday should be specified') + weekdays = [day.lower().strip() for day in weekdays] + for day in weekdays: + if day not in ALLOWED_WEEKDAYS: + raise InvalidScheduleException('Unknown weekday: ' + day) + return sorted(weekdays, key=lambda x: ALLOWED_WEEKDAYS.index(x)) + + +def read_schedule_config(incoming_schedule_config): + repeatable = read_repeatable_flag(incoming_schedule_config) + start_datetime = _read_start_datetime(incoming_schedule_config) + + prepared_schedule_config = ScheduleConfig(repeatable, start_datetime) + if repeatable: + prepared_schedule_config.repeat_unit = _read_repeat_unit(incoming_schedule_config) + prepared_schedule_config.repeat_period = _read_repeat_period(incoming_schedule_config) + + if prepared_schedule_config.repeat_unit == 'weeks': + prepared_schedule_config.weekdays = read_weekdays(incoming_schedule_config) + + return prepared_schedule_config + + +class ScheduleConfig: + + def __init__(self, repeatable, start_datetime) -> None: + self.repeatable = repeatable + self.start_datetime = start_datetime # type: datetime + self.repeat_unit = None + self.repeat_period = None + self.weekdays = None + + def as_serializable_dict(self): + result = { + 'repeatable': self.repeatable, + 'start_datetime': date_utils.to_iso_string(self.start_datetime) + } + + if self.repeat_unit is not None: + result['repeat_unit'] = self.repeat_unit + + if self.repeat_period is not None: + result['repeat_period'] = self.repeat_period + + if self.weekdays is not None: + result['weekdays'] = self.weekdays + + return result + + def get_next_time(self): + if not self.repeatable: + return self.start_datetime + + if self.repeat_unit == 'hours': + next_time_func = lambda start, iteration_index: start + timedelta( + hours=self.repeat_period * iteration_index) + + get_initial_multiplier = lambda start: \ + ((now - start).seconds // 3600 + (now - start).days * 24) \ + // self.repeat_period + elif self.repeat_unit == 'days': + next_time_func = lambda start, iteration_index: start + timedelta(days=self.repeat_period * iteration_index) + get_initial_multiplier = lambda start: (now - start).days // self.repeat_period + elif self.repeat_unit == 'months': + next_time_func = lambda start, iteration_index: date_utils.add_months(start, + self.repeat_period * iteration_index) + get_initial_multiplier = lambda start: (now - start).days // 28 // self.repeat_period + elif self.repeat_unit == 'weeks': + start_weekday = self.start_datetime.weekday() + offset = 0 + for weekday in self.weekdays: + index = ALLOWED_WEEKDAYS.index(weekday) + if index < start_weekday: + offset += 1 + + def next_weekday(start: datetime, iteration_index): + weeks_multiplier = (iteration_index + offset) // len(self.weekdays) + next_weekday_index = (iteration_index + offset) % len(self.weekdays) + next_weekday_name = self.weekdays[next_weekday_index] + next_weekday = ALLOWED_WEEKDAYS.index(next_weekday_name) + + return start \ + + timedelta(weeks=self.repeat_period * weeks_multiplier) \ + + timedelta(days=(next_weekday - start.weekday())) + + next_time_func = next_weekday + + get_initial_multiplier = lambda start: (now - start).days // 7 // self.repeat_period * len( + self.weekdays) - 1 + else: + raise Exception('Unknown unit: ' + repr(self.repeat_unit)) + + now = date_utils.now(tz=timezone.utc) + max_iterations = 10000 + initial_multiplier = max(0, get_initial_multiplier(self.start_datetime)) + i = 0 + while True: + resolved_time = next_time_func(self.start_datetime, i + initial_multiplier) + if resolved_time >= now: + return resolved_time + + i += 1 + if i > max_iterations: + raise Exception('Endless loop in calc next time') + + +class InvalidScheduleException(Exception): + def __init__(self, message) -> None: + super().__init__(message) diff --git a/src/scheduling/schedule_service.py b/src/scheduling/schedule_service.py new file mode 100644 index 00000000..8bc352b1 --- /dev/null +++ b/src/scheduling/schedule_service.py @@ -0,0 +1,177 @@ +import json +import logging +import os +import sched +import threading +import time +from datetime import timedelta + +from auth.user import User +from config.config_service import ConfigService +from execution.execution_service import ExecutionService +from execution.id_generator import IdGenerator +from scheduling import scheduling_job +from scheduling.schedule_config import read_schedule_config, InvalidScheduleException +from scheduling.scheduling_job import SchedulingJob +from utils import file_utils, date_utils + +SCRIPT_NAME_KEY = 'script_name' +USER_KEY = 'user' +PARAM_VALUES_KEY = 'parameter_values' + +JOB_SCHEDULE_KEY = 'schedule' + +LOGGER = logging.getLogger('script_server.scheduling.schedule_service') + +_sleep = time.sleep + + +def restore_jobs(schedules_folder): + files = [file for file in os.listdir(schedules_folder) if file.endswith('.json')] + + job_dict = {} + ids = [] # list of ALL ids, including broken configs + + for file in files: + try: + content = file_utils.read_file(os.path.join(schedules_folder, file)) + job_json = json.loads(content) + ids.append(job_json['id']) + + job = scheduling_job.from_dict(job_json) + + job_dict[job.id] = job + except: + LOGGER.exception('Failed to parse schedule file: ' + file) + + return job_dict, ids + + +class ScheduleService: + + def __init__(self, + config_service: ConfigService, + execution_service: ExecutionService, + conf_folder): + self._schedules_folder = os.path.join(conf_folder, 'schedules') + file_utils.prepare_folder(self._schedules_folder) + + self._config_service = config_service + self._execution_service = execution_service + + (jobs, ids) = restore_jobs(self._schedules_folder) + self._scheduled_executions = jobs + self._id_generator = IdGenerator(ids) + self.stopped = False + + self.scheduler = sched.scheduler(timefunc=time.time) + self._start_scheduler() + + for job in jobs.values(): + self.schedule_job(job) + + def create_job(self, script_name, parameter_values, incoming_schedule_config, user: User): + if user is None: + raise InvalidUserException('User id is missing') + + config_model = self._config_service.load_config_model(script_name, user, parameter_values) + self.validate_script_config(config_model) + + schedule_config = read_schedule_config(incoming_schedule_config) + + if not schedule_config.repeatable and date_utils.is_past(schedule_config.start_datetime): + raise InvalidScheduleException('Start date should be in the future') + + id = self._id_generator.next_id() + + job = SchedulingJob(id, user, schedule_config, script_name, parameter_values) + + self.save_job(job) + + self.schedule_job(job) + + return id + + @staticmethod + def validate_script_config(config_model): + if not config_model.schedulable: + raise UnavailableScriptException(config_model.name + ' is not schedulable') + + for parameter in config_model.parameters: + if parameter.secure: + raise UnavailableScriptException( + 'Script contains secure parameters (' + parameter.str_name() + '), this is not supported') + + def schedule_job(self, job: SchedulingJob): + schedule = job.schedule + + if not schedule.repeatable and date_utils.is_past(schedule.start_datetime): + return + + next_datetime = schedule.get_next_time() + LOGGER.info( + 'Scheduling ' + job.get_log_name() + ' at ' + next_datetime.astimezone(tz=None).strftime('%H:%M, %d %B %Y')) + + self.scheduler.enterabs(next_datetime.timestamp(), 1, self._execute_job, (job,)) + + def _execute_job(self, job: SchedulingJob): + LOGGER.info('Executing ' + job.get_log_name()) + + script_name = job.script_name + parameter_values = job.parameter_values + user = job.user + + try: + config = self._config_service.load_config_model(script_name, user, parameter_values) + self.validate_script_config(config) + + execution_id = self._execution_service.start_script(config, parameter_values, user.user_id, + user.audit_names) + LOGGER.info('Started script #' + str(execution_id) + ' for ' + job.get_log_name()) + except: + LOGGER.exception('Failed to execute ' + job.get_log_name()) + + self.schedule_job(job) + + def save_job(self, job: SchedulingJob): + user = job.user + script_name = job.script_name + + filename = file_utils.to_filename('%s_%s_%s.json' % (script_name, user.get_audit_name(), job.id)) + file_utils.write_file( + os.path.join(self._schedules_folder, filename), + json.dumps(job.as_serializable_dict(), indent=2)) + + def _start_scheduler(self): + def scheduler_loop(): + while not self.stopped: + try: + self.scheduler.run(blocking=False) + except: + LOGGER.exception('Failed to execute scheduled job') + + now = date_utils.now() + sleep_delta = timedelta(minutes=1) - timedelta(microseconds=now.microsecond, seconds=now.second) + _sleep(sleep_delta.total_seconds()) + + self.scheduling_thread = threading.Thread(daemon=True, target=scheduler_loop) + self.scheduling_thread.start() + + def _stop(self): + self.stopped = True + + def stopper(): + pass + + # just schedule the next execution to exit thread immediately + self.scheduler.enter(1, 0, stopper) + + +class InvalidUserException(Exception): + def __init__(self, message) -> None: + super().__init__(message) + + +class UnavailableScriptException(Exception): + def __init__(self, message) -> None: + super().__init__(message) diff --git a/src/scheduling/scheduling_job.py b/src/scheduling/scheduling_job.py new file mode 100644 index 00000000..cdccafba --- /dev/null +++ b/src/scheduling/scheduling_job.py @@ -0,0 +1,35 @@ +from auth import user +from auth.user import User +from scheduling import schedule_config +from scheduling.schedule_config import ScheduleConfig + + +class SchedulingJob: + def __init__(self, id, user, schedule_config, script_name, parameter_values) -> None: + self.id = str(id) + self.user = user # type: User + self.schedule = schedule_config # type: ScheduleConfig + self.script_name = script_name + self.parameter_values = parameter_values # type: dict + + def as_serializable_dict(self): + return { + 'id': self.id, + 'user': self.user.as_serializable_dict(), + 'schedule': self.schedule.as_serializable_dict(), + 'script_name': self.script_name, + 'parameter_values': self.parameter_values + } + + def get_log_name(self): + return 'Job#' + str(self.id) + '-' + self.script_name + + +def from_dict(job_as_dict): + id = job_as_dict['id'] + parsed_user = user.from_serialized_dict(job_as_dict['user']) + schedule = schedule_config.read_schedule_config(job_as_dict['schedule']) + script_name = job_as_dict['script_name'] + parameter_values = job_as_dict['parameter_values'] + + return SchedulingJob(id, parsed_user, schedule, script_name, parameter_values) diff --git a/src/tests/__init__.py b/src/tests/__init__.py old mode 100755 new mode 100644 diff --git a/src/tests/date_utils_test.py b/src/tests/date_utils_test.py index d4b8183c..a0af980c 100644 --- a/src/tests/date_utils_test.py +++ b/src/tests/date_utils_test.py @@ -1,5 +1,5 @@ import unittest -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from utils import date_utils @@ -20,3 +20,168 @@ def test_astimezone_naive_before_dst(self): transformed_datetime = date_utils.astimezone(naive_datetime, timezone.utc) self.assertEqual(utc_datetime, transformed_datetime) + + +class TestParseIsoDatetime(unittest.TestCase): + def test_parse_correct_time(self): + parsed = date_utils.parse_iso_datetime('2020-07-10T15:30:59.123456Z') + expected = datetime(2020, 7, 10, 15, 30, 59, 123456, timezone.utc) + self.assertEqual(expected, parsed) + + def test_parse_wrong_time(self): + self.assertRaisesRegex( + ValueError, + 'does not match format', + date_utils.parse_iso_datetime, + '15:30:59 2020-07-10') + + +class TestToIsoString(unittest.TestCase): + def test_utc_time(self): + iso_string = date_utils.to_iso_string(datetime(2020, 7, 10, 15, 30, 59, 123456, timezone.utc)) + self.assertEqual('2020-07-10T15:30:59.123456Z', iso_string) + + def test_naive_time(self): + iso_string = date_utils.to_iso_string(datetime(2020, 7, 10, 15, 30, 59, 123456)) + self.assertEqual('2020-07-10T15:30:59.123456Z', iso_string) + + def test_local_time(self): + iso_string = date_utils.to_iso_string(datetime(2020, 7, 10, 15, 30, 59, 123456, timezone(timedelta(hours=1)))) + self.assertEqual('2020-07-10T14:30:59.123456Z', iso_string) + + +class TestIsPast(unittest.TestCase): + def test_when_past_naive(self): + value = datetime(2020, 7, 10, 15, 30, 59, 123456) + + self.assertTrue(date_utils.is_past(value)) + + def test_when_past_utc(self): + value = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + + self.assertTrue(date_utils.is_past(value)) + + def test_when_future_naive(self): + value = datetime(2030, 7, 10, 15, 30, 59, 123456) + + self.assertFalse(date_utils.is_past(value)) + + def test_when_future_utc(self): + value = datetime(2030, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + + self.assertFalse(date_utils.is_past(value)) + + def test_when_now(self): + value = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + + date_utils._mocked_now = value + + self.assertFalse(date_utils.is_past(value)) + + def tearDown(self) -> None: + date_utils._mocked_now = None + + +class TestSecondsBetween(unittest.TestCase): + def test_small_positive_delta(self): + start = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + end = datetime(2020, 7, 10, 15, 33, 12, 123456, tzinfo=timezone.utc) + + seconds = date_utils.seconds_between(start, end) + self.assertEqual(133, seconds) + + def test_small_negative_delta(self): + start = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + end = datetime(2020, 7, 10, 15, 30, 13, 123456, tzinfo=timezone.utc) + + seconds = date_utils.seconds_between(start, end) + self.assertEqual(-46, seconds) + + def test_large_positive_delta(self): + start = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + end = datetime(2021, 2, 15, 17, 33, 12, 123456, tzinfo=timezone.utc) + + seconds = date_utils.seconds_between(start, end) + self.assertEqual(19015333, seconds) + + def test_large_negative_delta(self): + start = datetime(2020, 7, 10, 15, 30, 13, 123456, tzinfo=timezone.utc) + end = datetime(2019, 11, 29, 9, 30, 59, 123456, tzinfo=timezone.utc) + + seconds = date_utils.seconds_between(start, end) + self.assertEqual(-19375154, seconds) + + def test_delta_with_microseconds(self): + start = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + end = datetime(2020, 7, 10, 15, 33, 12, 876543, tzinfo=timezone.utc) + + seconds = date_utils.seconds_between(start, end) + self.assertEqual(133.753087, seconds) + + +class TestAddMonths(unittest.TestCase): + def test_add_one_month(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 1) + expected = datetime(2020, 8, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_4_months(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 4) + expected = datetime(2020, 11, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_roll_next_year(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 6) + expected = datetime(2021, 1, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_roll_multiple_years(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 33) + expected = datetime(2023, 4, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_last_day_when_next_shorter(self): + original = datetime(2020, 7, 31, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 2) + expected = datetime(2020, 9, 30, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_last_day_when_next_same(self): + original = datetime(2020, 7, 31, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 1) + expected = datetime(2020, 8, 31, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_last_day_when_next_longer(self): + original = datetime(2020, 6, 30, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 2) + expected = datetime(2020, 8, 30, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_last_day_when_next_february(self): + original = datetime(2020, 7, 30, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 7) + expected = datetime(2021, 2, 28, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_add_months_to_last_day_when_next_leap_february(self): + original = datetime(2019, 7, 30, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, 7) + expected = datetime(2020, 2, 29, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_subtract_one_month(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, -1) + expected = datetime(2020, 6, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) + + def test_subtract_months_to_prev_year(self): + original = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + added = date_utils.add_months(original, -10) + expected = datetime(2019, 9, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected, added) diff --git a/src/tests/external_model_test.py b/src/tests/external_model_test.py index 03c11a9c..f7e5ccd1 100644 --- a/src/tests/external_model_test.py +++ b/src/tests/external_model_test.py @@ -3,7 +3,8 @@ from datetime import datetime, timezone from execution.logging import HistoryEntry -from model.external_model import to_short_execution_log, to_long_execution_log, server_conf_to_external +from model.external_model import to_short_execution_log, to_long_execution_log, server_conf_to_external, \ + parse_external_schedule from model.server_conf import ServerConfig @@ -163,3 +164,43 @@ def test_config_with_none_values(self): self.assertIsNone(external_config.get('title')) self.assertIsNone(external_config.get('enableScriptTitles')) self.assertIsNone(external_config.get('version')) + + +class TestParseExternalSchedule(unittest.TestCase): + def test_parse_full_config(self): + parsed = parse_external_schedule( + {'repeatable': False, 'startDatetime': '2020-12-30', 'repeatUnit': 'days', 'repeatPeriod': 5, + 'weekDays': ['monday', 'Tuesday']}) + + self.assertDictEqual({ + 'repeatable': False, + 'start_datetime': '2020-12-30', + 'repeat_unit': 'days', + 'repeat_period': 5, + 'weekdays': ['monday', 'Tuesday']}, + parsed) + + def test_parse_partial_config(self): + parsed = parse_external_schedule( + {'repeatable': False, 'startDatetime': '2020-12-30'}) + + self.assertDictEqual({ + 'repeatable': False, + 'start_datetime': '2020-12-30', + 'repeat_unit': None, + 'repeat_period': None, + 'weekdays': None}, + parsed) + + def test_parse_unknown_field(self): + parsed = parse_external_schedule( + {'repeatable': False, + 'startDatetime': '2020-12-30', + 'anotherField': 'abc'}) + + self.assertDictEqual({ + 'repeatable': False, + 'start_datetime': '2020-12-30', + 'repeat_unit': None, + 'repeat_period': None, + 'weekdays': None}, parsed) diff --git a/src/tests/file_utils_test.py b/src/tests/file_utils_test.py new file mode 100644 index 00000000..5b5cac00 --- /dev/null +++ b/src/tests/file_utils_test.py @@ -0,0 +1,20 @@ +from unittest import TestCase + +from utils import os_utils, file_utils + + +class TestToFilename(TestCase): + def test_replace_special_characters_linux(self): + os_utils.set_linux() + + filename = file_utils.to_filename('!@#$%^&*()_+\|/?.<>,\'"') + self.assertEqual('!@#$%^&*()_+\\|_?.<>,\'"', filename) + + def test_replace_special_characters_windows(self): + os_utils.set_win() + + filename = file_utils.to_filename('!@#$%^&*()_+\|/?.<>,\'"') + self.assertEqual('!@#$%^&_()_+____.__,\'_', filename) + + def tearDown(self) -> None: + os_utils.reset_os() diff --git a/src/tests/model_helper_test.py b/src/tests/model_helper_test.py index f718b37e..ad231f0a 100644 --- a/src/tests/model_helper_test.py +++ b/src/tests/model_helper_test.py @@ -1,5 +1,6 @@ import os import unittest +from datetime import datetime, timezone from config.constants import FILE_TYPE_FILE, FILE_TYPE_DIR from model import model_helper @@ -400,3 +401,39 @@ def test_text_when_blank_to_none_and_blank_and_default(self): def test_text_when_int(self): self.assertRaisesRegex(InvalidValueTypeException, 'Invalid key1 value: string expected, but was: 5', read_str_from_config, {'key1': 5}, 'key1') + + +class TestReadDatetime(unittest.TestCase): + def test_datetime_value(self): + value = datetime.now() + actual_value = model_helper.read_datetime_from_config('p1', {'p1': value}) + self.assertEqual(value, actual_value) + + def test_string_value(self): + actual_value = model_helper.read_datetime_from_config('p1', {'p1': '2020-07-10T15:30:59.123456Z'}) + expected_value = datetime(2020, 7, 10, 15, 30, 59, 123456, tzinfo=timezone.utc) + self.assertEqual(expected_value, actual_value) + + def test_string_value_when_bad_format(self): + self.assertRaisesRegex( + ValueError, + 'does not match format', + model_helper.read_datetime_from_config, + 'p1', {'p1': '15:30:59 2020-07-10'}) + + def test_default_value_when_missing_key(self): + value = datetime.now() + actual_value = model_helper.read_datetime_from_config('p1', {'another_key': 'abc'}, default=value) + self.assertEqual(value, actual_value) + + def test_default_value_when_value_none(self): + value = datetime.now() + actual_value = model_helper.read_datetime_from_config('p1', {'p1': None}, default=value) + self.assertEqual(value, actual_value) + + def test_int_value(self): + self.assertRaisesRegex( + InvalidValueTypeException, + 'should be a datetime', + model_helper.read_datetime_from_config, + 'p1', {'p1': 12345}) diff --git a/src/tests/scheduling/__init__.py b/src/tests/scheduling/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/scheduling/schedule_config_test.py b/src/tests/scheduling/schedule_config_test.py new file mode 100644 index 00000000..e7499968 --- /dev/null +++ b/src/tests/scheduling/schedule_config_test.py @@ -0,0 +1,128 @@ +from unittest import TestCase + +from parameterized import parameterized + +from scheduling.schedule_config import ScheduleConfig +from utils import date_utils + + +def to_datetime(short_datetime_string): + dt_string = short_datetime_string + ':0.000000Z' + return date_utils.parse_iso_datetime(dt_string.replace(' ', 'T')) + + +class TestGetNextTime(TestCase): + @parameterized.expand([ + ('2020-03-19 11:30', '2020-03-15 16:13', 1, 'days', '2020-03-19 16:13'), + ('2020-03-19 17:30', '2020-03-15 16:13', 1, 'days', '2020-03-20 16:13'), + ('2020-03-15 11:30', '2020-03-15 16:13', 1, 'days', '2020-03-15 16:13'), + ('2020-03-14 11:30', '2020-03-15 16:13', 1, 'days', '2020-03-15 16:13'), + ('2020-03-15 16:13', '2020-03-15 16:13', 1, 'days', '2020-03-15 16:13'), + ('2020-03-15 16:14', '2020-03-15 16:13', 1, 'days', '2020-03-16 16:13'), + ('2020-03-19 11:30', '2020-03-15 16:13', 2, 'days', '2020-03-19 16:13'), + ('2020-03-20 11:30', '2020-03-15 16:13', 2, 'days', '2020-03-21 16:13'), + ('2020-03-19 16:13', '2020-03-15 16:13', 2, 'days', '2020-03-19 16:13'), + ('2020-03-18 11:30', '2020-03-15 16:13', 5, 'days', '2020-03-20 16:13'), + ('2020-03-20 11:30', '2020-03-15 16:13', 24, 'days', '2020-04-08 16:13'), + ('2020-04-09 11:30', '2020-03-15 16:13', 24, 'days', '2020-05-02 16:13'), + ('2020-03-19 11:30', '2020-03-15 16:13', 1, 'hours', '2020-03-19 12:13'), + ('2020-03-19 17:30', '2020-03-15 16:13', 1, 'hours', '2020-03-19 18:13'), + ('2020-03-15 11:30', '2020-03-15 16:13', 1, 'hours', '2020-03-15 16:13'), + ('2020-03-14 11:30', '2020-03-15 16:13', 1, 'hours', '2020-03-15 16:13'), + ('2020-03-15 16:13', '2020-03-15 16:13', 1, 'hours', '2020-03-15 16:13'), + ('2020-03-15 16:14', '2020-03-15 16:13', 1, 'hours', '2020-03-15 17:13'), + # big difference between start and now + ('2023-08-29 16:14', '2020-03-15 16:13', 1, 'hours', '2023-08-29 17:13'), + ('2020-03-19 10:30', '2020-03-15 16:13', 2, 'hours', '2020-03-19 12:13'), + ('2020-03-19 11:30', '2020-03-15 16:13', 2, 'hours', '2020-03-19 12:13'), + ('2020-03-19 16:13', '2020-03-15 16:13', 2, 'hours', '2020-03-19 16:13'), + ('2020-03-18 11:30', '2020-03-15 16:13', 5, 'hours', '2020-03-18 14:13'), + ('2020-03-20 11:30', '2020-03-15 16:13', 24, 'hours', '2020-03-20 16:13'), + ('2020-04-09 17:30', '2020-03-15 16:13', 24, 'hours', '2020-04-10 16:13'), + ('2020-03-19 11:30', '2020-03-15 16:13', 1, 'months', '2020-04-15 16:13'), + ('2020-03-19 17:30', '2020-03-15 16:13', 1, 'months', '2020-04-15 16:13'), + ('2020-03-15 11:30', '2020-03-15 16:13', 1, 'months', '2020-03-15 16:13'), + ('2020-03-14 11:30', '2020-03-15 16:13', 1, 'months', '2020-03-15 16:13'), + ('2020-03-15 16:13', '2020-03-15 16:13', 1, 'months', '2020-03-15 16:13'), + ('2020-03-15 16:14', '2020-03-15 16:13', 1, 'months', '2020-04-15 16:13'), + ('2020-04-01 16:11', '2020-03-31 16:13', 1, 'months', '2020-04-30 16:13'), + ('2021-01-31 20:00', '2021-01-31 16:13', 1, 'months', '2021-02-28 16:13'), # Roll to February + ('2020-01-31 20:00', '2020-01-31 16:13', 1, 'months', '2020-02-29 16:13'), # Roll to February leap year + ('2020-03-19 10:30', '2020-03-15 16:13', 2, 'months', '2020-05-15 16:13'), + ('2020-04-19 11:30', '2020-03-15 16:13', 2, 'months', '2020-05-15 16:13'), + ('2020-03-15 16:13', '2020-03-15 16:13', 2, 'months', '2020-03-15 16:13'), + ('2020-04-01 16:11', '2020-03-31 16:13', 2, 'months', '2020-05-31 16:13'), + ('2020-03-18 11:30', '2020-03-15 16:13', 5, 'months', '2020-08-15 16:13'), + ('2020-08-18 11:30', '2020-03-15 16:13', 5, 'months', '2021-01-15 16:13'), + ('2021-01-18 11:30', '2020-03-15 16:13', 5, 'months', '2021-06-15 16:13'), + ('2020-03-16 11:30', '2020-03-15 16:13', 13, 'months', '2021-04-15 16:13'), + ('2020-03-19 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-20 16:13', ['monday', 'friday']), + ('2020-03-15 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-16 16:13', ['monday', 'friday']), + ('2020-03-16 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-16 16:13', ['monday', 'friday']), + ('2020-03-16 16:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-20 16:13', ['monday', 'friday']), + ('2020-03-20 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-20 16:13', ['monday', 'friday']), + ('2020-04-04 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-06 16:13', ['monday', 'friday']), + ('2020-04-07 11:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'friday']), + ('2020-03-16 16:13', '2020-03-16 16:13', 1, 'weeks', '2020-03-16 16:13', ['monday', 'friday']), + ('2020-03-16 16:14', '2020-03-16 16:13', 1, 'weeks', '2020-03-20 16:13', ['monday', 'friday']), + # Test for testing start date on different weekdays, now tuesday + ('2020-04-07 1:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 2:30', '2020-03-16 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 3:30', '2020-03-17 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 4:30', '2020-03-18 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 5:30', '2020-03-19 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 6:30', '2020-03-20 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-07 7:30', '2020-03-21 16:13', 1, 'weeks', '2020-04-08 16:13', ['monday', 'wednesday', 'friday']), + # Test for testing start date on different weekdays, now thursday + ('2020-04-09 1:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 2:30', '2020-03-16 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 3:30', '2020-03-17 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 4:30', '2020-03-18 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 5:30', '2020-03-19 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 6:30', '2020-03-20 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-09 7:30', '2020-03-21 16:13', 1, 'weeks', '2020-04-10 16:13', ['monday', 'wednesday', 'friday']), + # Test for testing start date on different weekdays, now saturday + ('2020-04-11 1:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 2:30', '2020-03-16 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 3:30', '2020-03-17 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 4:30', '2020-03-18 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 5:30', '2020-03-19 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 6:30', '2020-03-20 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-11 7:30', '2020-03-21 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + # Test for testing start date on different weekdays, now monday + ('2020-04-13 1:30', '2020-03-15 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 2:30', '2020-03-16 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 3:30', '2020-03-17 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 4:30', '2020-03-18 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 5:30', '2020-03-19 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 6:30', '2020-03-20 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + ('2020-04-13 7:30', '2020-03-21 16:13', 1, 'weeks', '2020-04-13 16:13', ['monday', 'wednesday', 'friday']), + # Test for testing start date on different weekdays, now wednesday, when larger interval + ('2020-09-16 1:30', '2020-03-14 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 2:30', '2020-03-15 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 3:30', '2020-03-16 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 4:30', '2020-03-17 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 5:30', '2020-03-18 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 6:30', '2020-03-19 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-09-16 7:30', '2020-03-20 16:13', 1, 'weeks', '2020-09-19 16:13', ['tuesday', 'saturday']), + ('2020-03-16 16:30', '2020-03-15 16:13', 1, 'weeks', '2020-03-18 16:13', ['wednesday']), + ('2020-03-19 11:30', '2020-03-15 16:13', 2, 'weeks', '2020-03-23 16:13', ['monday', 'friday']), + ('2020-03-24 11:30', '2020-03-15 16:13', 2, 'weeks', '2020-03-27 16:13', ['monday', 'friday']), + ('2020-06-07 17:30', '2020-03-15 16:13', 2, 'weeks', '2020-06-15 16:13', ['monday', 'friday']), + ('2020-06-07 17:30', '2020-03-15 16:13', 2, 'weeks', '2020-06-16 16:13', ['tuesday', 'wednesday']), + ]) + def test_next_day_when_repeatable(self, now_dt, start, period, unit, expected, weekdays=None): + date_utils._mocked_now = to_datetime(now_dt) + + config = ScheduleConfig(True, to_datetime(start)) + config.repeat_period = period + config.repeat_unit = unit + config.weekdays = weekdays + + next_time = config.get_next_time() + self.assertEqual(to_datetime(expected), next_time) + + def tearDown(self) -> None: + super().tearDown() + + date_utils._mocked_now = None diff --git a/src/tests/scheduling/schedule_service_test.py b/src/tests/scheduling/schedule_service_test.py new file mode 100644 index 00000000..1ecff2b9 --- /dev/null +++ b/src/tests/scheduling/schedule_service_test.py @@ -0,0 +1,369 @@ +import json +import os +import time +from datetime import timedelta +from typing import Sequence +from unittest import TestCase +from unittest.mock import patch, ANY, MagicMock + +from auth.user import User +from scheduling import schedule_service +from scheduling.schedule_config import ScheduleConfig, InvalidScheduleException +from scheduling.schedule_service import ScheduleService, InvalidUserException, UnavailableScriptException +from scheduling.scheduling_job import SchedulingJob +from tests import test_utils +from utils import date_utils, audit_utils, file_utils + +mocked_now = date_utils.parse_iso_datetime('2020-07-24T12:30:59.000000Z') +mocked_now_epoch = mocked_now.timestamp() + + +class ScheduleServiceTestCase(TestCase): + def assert_schedule_calls(self, expected_job_time_pairs): + self.assertEqual(len(expected_job_time_pairs), len(self.scheduler_mock.enterabs.call_args_list)) + + for i, pair in enumerate(expected_job_time_pairs): + expected_time = date_utils.sec_to_datetime(pair[1]) + expected_job = pair[0] + + # the first item of call_args is actual arguments, passed to the method + args = self.scheduler_mock.enterabs.call_args_list[i][0] + + # we schedule job as enterabs(expected_time, priority, self._execute_job, (job,)) + # to get the job, we need to get the last arg, and extract the first parameter from it + schedule_method_args_tuple = args[3] + schedule_method_job_arg = schedule_method_args_tuple[0] + actual_time = date_utils.sec_to_datetime(args[0]) + + self.assertEqual(expected_time, actual_time) + self.assertDictEqual(expected_job.as_serializable_dict(), + schedule_method_job_arg.as_serializable_dict()) + + def mock_schedule_model_with_secure_param(self): + model = test_utils.create_config_model('some-name', parameters=[{'name': 'p1', 'secure': True}]) + model.schedulable = True + + self.config_service.load_config_model.side_effect = lambda a, b, c: model + + def setUp(self) -> None: + super().setUp() + + self.patcher = patch('sched.scheduler') + self.scheduler_mock = MagicMock() + self.patcher.start().return_value = self.scheduler_mock + + schedule_service._sleep = MagicMock() + schedule_service._sleep.side_effect = lambda x: time.sleep(0.001) + + self.unschedulable_scripts = set() + self.config_service = MagicMock() + self.config_service.load_config_model.side_effect = lambda name, b, c: test_utils.create_config_model( + name, + schedulable=name not in self.unschedulable_scripts) + + self.execution_service = MagicMock() + + self.schedule_service = ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + + date_utils._mocked_now = mocked_now + + test_utils.setup() + + def tearDown(self) -> None: + super().tearDown() + + test_utils.cleanup() + + date_utils._mocked_now = None + + self.schedule_service._stop() + self.schedule_service.scheduling_thread.join() + + schedule_service._sleep = time.sleep + + self.patcher.stop() + + +class TestScheduleServiceCreateJob(ScheduleServiceTestCase): + def test_create_job_when_single(self): + job_prototype = create_job() + job_id = self.call_create_job(job_prototype) + + self.assertEqual('1', job_id) + + job_prototype.id = job_id + self.verify_config_files([job_prototype]) + + def test_create_job_when_multiple(self): + jobs = [] + + for i in range(1, 3): + job_prototype = create_job( + user_id='user-' + str(i), + script_name='script-' + str(i), + repeatable=i % 2 == 1, + parameter_values={'p1': 'hi', 'p2': i}) + job_id = self.call_create_job(job_prototype) + + self.assertEqual(str(i), job_id) + + job_prototype.id = job_id + + jobs.append(job_prototype) + + self.verify_config_files(jobs) + + def test_create_job_when_user_none(self): + self.assertRaisesRegex( + InvalidUserException, + 'User id is missing', + self.schedule_service.create_job, + 'abc', {}, {}, None) + + def test_create_job_when_not_schedulable(self): + job_prototype = create_job() + self.unschedulable_scripts.add('my_script_A') + + self.assertRaisesRegex( + UnavailableScriptException, + 'is not schedulable', + self.call_create_job, + job_prototype) + + def test_create_job_when_secure(self): + self.mock_schedule_model_with_secure_param() + + job_prototype = create_job() + + self.assertRaisesRegex( + UnavailableScriptException, + 'Script contains secure parameters', + self.call_create_job, + job_prototype) + + def test_create_job_when_non_repeatable_in_the_past(self): + job_prototype = create_job(repeatable=False, start_datetime=mocked_now - timedelta(seconds=1)) + + self.assertRaisesRegex( + InvalidScheduleException, + 'Start date should be in the future', + self.call_create_job, + job_prototype) + + def test_create_job_verify_scheduler_call_when_one_time(self): + job_prototype = create_job(id='1', repeatable=False, start_datetime=mocked_now + timedelta(seconds=97)) + self.call_create_job(job_prototype) + + self.assert_schedule_calls([(job_prototype, mocked_now_epoch + 97)]) + + def test_create_job_verify_timer_call_when_repeatable(self): + job_prototype = create_job(id='1', repeatable=True, start_datetime=mocked_now - timedelta(seconds=97)) + self.call_create_job(job_prototype) + + self.assert_schedule_calls([(job_prototype, mocked_now_epoch + 1468703)]) + + def call_create_job(self, job: SchedulingJob): + return self.schedule_service.create_job( + job.script_name, + job.parameter_values, + job.schedule.as_serializable_dict(), + job.user) + + def verify_config_files(self, expected_jobs: Sequence[SchedulingJob]): + expected_files = [get_job_filename(job) for job in expected_jobs] + + schedules_dir = os.path.join(test_utils.temp_folder, 'schedules') + test_utils.assert_dir_files(expected_files, schedules_dir, self) + + for job in expected_jobs: + job_path = os.path.join(schedules_dir, get_job_filename(job)) + content = file_utils.read_file(job_path) + restored_job = json.loads(content) + + self.assertEqual(restored_job, job.as_serializable_dict()) + + +class TestScheduleServiceInit(ScheduleServiceTestCase): + def test_no_config_folder(self): + test_utils.cleanup() + + schedule_service = ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assertEqual(schedule_service._scheduled_executions, {}) + self.assertEqual('1', schedule_service._id_generator.next_id()) + + def test_restore_multiple_configs(self): + job1 = create_job(id='11') + job2 = create_job(id=9) + job3 = create_job(id=3) + self.save_job(job1) + self.save_job(job2) + self.save_job(job3) + + schedule_service = ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assertSetEqual({'3', '9', '11'}, set(schedule_service._scheduled_executions.keys())) + self.assertEqual('12', schedule_service._id_generator.next_id()) + + def test_restore_configs_when_one_corrupted(self): + job1 = create_job(id='11', repeatable=None) + job2 = create_job(id=3) + self.save_job(job1) + self.save_job(job2) + + schedule_service = ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assertSetEqual({'3'}, set(schedule_service._scheduled_executions.keys())) + self.assertEqual('12', schedule_service._id_generator.next_id()) + + def test_schedule_on_restore_when_one_time(self): + job = create_job(id=3, repeatable=False, start_datetime=mocked_now + timedelta(minutes=3)) + self.save_job(job) + + ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assert_schedule_calls([(job, mocked_now_epoch + 180)]) + + def test_schedule_on_restore_when_one_time_in_past(self): + job = create_job(id=3, repeatable=False, start_datetime=mocked_now - timedelta(seconds=1)) + self.save_job(job) + + ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assert_schedule_calls([]) + + def test_schedule_on_restore_when_repeatable_in_future(self): + job = create_job(id=3, repeatable=True, start_datetime=mocked_now + timedelta(hours=3)) + self.save_job(job) + + ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assert_schedule_calls([(job, mocked_now_epoch + 1479600)]) + + def test_schedule_on_restore_when_repeatable_in_past(self): + job = create_job(id=3, repeatable=True, start_datetime=mocked_now + timedelta(days=2)) + self.save_job(job) + + ScheduleService(self.config_service, self.execution_service, test_utils.temp_folder) + self.assert_schedule_calls([(job, mocked_now_epoch + 1468800)]) + + def test_scheduler_runner(self): + original_runs_count = self.scheduler_mock.run.call_count + time.sleep(0.1) + step1_runs_count = self.scheduler_mock.run.call_count + + self.assertGreater(step1_runs_count, original_runs_count) + + time.sleep(0.1) + step2_runs_count = self.scheduler_mock.run.call_count + self.assertGreater(step2_runs_count, step1_runs_count) + + def test_scheduler_runner_when_stopped(self): + self.schedule_service._stop() + time.sleep(0.1) + original_runs_count = self.scheduler_mock.run.call_count + + time.sleep(0.1) + + final_runs_count = self.scheduler_mock.run.call_count + self.assertEqual(final_runs_count, original_runs_count) + + def save_job(self, job): + schedules_dir = os.path.join(test_utils.temp_folder, 'schedules') + path = os.path.join(schedules_dir, get_job_filename(job)) + content = json.dumps(job.as_serializable_dict()) + file_utils.write_file(path, content) + + +class TestScheduleServiceExecuteJob(ScheduleServiceTestCase): + def test_execute_simple_job(self): + job = create_job(id=1, repeatable=False, start_datetime=mocked_now - timedelta(seconds=1)) + + self.schedule_service._execute_job(job) + + self.execution_service.start_script.assert_called_once_with( + ANY, job.parameter_values, job.user.user_id, job.user.audit_names) + self.assert_schedule_calls([]) + + def test_execute_repeatable_job(self): + job = create_job(id=1, + repeatable=True, + start_datetime=mocked_now - timedelta(seconds=1), + repeat_unit='days', + repeat_period=1) + + self.schedule_service._execute_job(job) + + self.execution_service.start_script.assert_called_once_with( + ANY, job.parameter_values, job.user.user_id, job.user.audit_names) + self.assert_schedule_calls([(job, mocked_now_epoch + 86399)]) + + def test_execute_when_fails(self): + job = create_job(id=1, + repeatable=True, + start_datetime=mocked_now - timedelta(seconds=1), + repeat_unit='days', + repeat_period=1) + + self.execution_service.start_script.side_effect = Exception('Test exception') + self.schedule_service._execute_job(job) + + self.assert_schedule_calls([(job, mocked_now_epoch + 86399)]) + + def test_execute_when_not_schedulable(self): + job = create_job(id=1, + repeatable=True, + start_datetime=mocked_now - timedelta(seconds=1), + repeat_unit='days', + repeat_period=1) + + self.unschedulable_scripts.add(job.script_name) + + self.schedule_service._execute_job(job) + + self.execution_service.start_script.assert_not_called() + self.assert_schedule_calls([(job, mocked_now_epoch + 86399)]) + + def test_execute_when_has_secure_parameters(self): + job = create_job(id=1, + repeatable=True, + start_datetime=mocked_now - timedelta(seconds=1), + repeat_unit='days', + repeat_period=1) + + self.mock_schedule_model_with_secure_param() + + self.schedule_service._execute_job(job) + + self.execution_service.start_script.assert_not_called() + self.assert_schedule_calls([(job, mocked_now_epoch + 86399)]) + + +def create_job(id=None, + user_id='UserX', + script_name='my_script_A', + audit_names=None, + repeatable=True, + start_datetime=mocked_now + timedelta(seconds=5), + repeat_unit=None, + repeat_period=None, + weekdays=None, + parameter_values=None): + if audit_names is None: + audit_names = {audit_utils.HOSTNAME: 'my-host'} + + if repeatable and repeat_unit is None: + repeat_unit = 'weeks' + if repeatable and repeat_period is None: + repeat_period = 3 + + if weekdays is None and repeatable and repeat_unit == 'weeks': + weekdays = ['monday', 'wednesday'] + + if parameter_values is None: + parameter_values = {'p1': 987, 'param_2': ['hello', 'world']} + + schedule_config = ScheduleConfig(repeatable, start_datetime) + schedule_config.repeat_unit = repeat_unit + schedule_config.repeat_period = repeat_period + schedule_config.weekdays = weekdays + + return SchedulingJob(id, User(user_id, audit_names), schedule_config, script_name, parameter_values) + + +def get_job_filename(job): + return job.script_name + '_' + job.user.get_audit_name() + '_' + str(job.id) + '.json' diff --git a/src/tests/scheduling/scheduling_job_test.py b/src/tests/scheduling/scheduling_job_test.py new file mode 100644 index 00000000..c75a86f3 --- /dev/null +++ b/src/tests/scheduling/scheduling_job_test.py @@ -0,0 +1,36 @@ +import json +from datetime import datetime, timezone +from unittest import TestCase + +from auth.user import User +from scheduling.schedule_config import ScheduleConfig +from scheduling.scheduling_job import SchedulingJob, from_dict +from utils import audit_utils + + +class TestSchedulingJob(TestCase): + def test_serialize_deserialize(self): + user = User('user-X', {audit_utils.AUTH_USERNAME: 'user-X', audit_utils.HOSTNAME: 'localhost'}) + schedule_config = ScheduleConfig(True, start_datetime=datetime.now(tz=timezone.utc)) + schedule_config.repeat_unit = 'weeks' + schedule_config.repeat_period = 3 + schedule_config.weekdays = ['monday', 'wednesday'] + parameter_values = {'p1': 9, 'p2': ['A', 'C']} + + job = SchedulingJob(123, user, schedule_config, 'my_script', parameter_values) + + serialized = json.dumps(job.as_serializable_dict()) + restored_job = from_dict(json.loads(serialized)) + + self.assertEqual(job.id, restored_job.id) + self.assertEqual(job.script_name, restored_job.script_name) + self.assertEqual(job.parameter_values, restored_job.parameter_values) + + self.assertEqual(job.user.user_id, restored_job.user.user_id) + self.assertEqual(job.user.audit_names, restored_job.user.audit_names) + + self.assertEqual(job.schedule.repeatable, restored_job.schedule.repeatable) + self.assertEqual(job.schedule.start_datetime, restored_job.schedule.start_datetime) + self.assertEqual(job.schedule.repeat_period, restored_job.schedule.repeat_period) + self.assertEqual(job.schedule.repeat_unit, restored_job.schedule.repeat_unit) + self.assertEqual(job.schedule.weekdays, restored_job.schedule.weekdays) diff --git a/src/tests/script_config_test.py b/src/tests/script_config_test.py index 8d9eeded..eef59aa6 100644 --- a/src/tests/script_config_test.py +++ b/src/tests/script_config_test.py @@ -34,7 +34,8 @@ def test_create_full_config(self): 'working_directory': working_directory, 'requires_terminal': requires_terminal, 'bash_formatting': bash_formatting, - 'output_files': output_files}) + 'output_files': output_files, + 'scheduling': {'enabled': True}}) self.assertEqual(name, config_model.name) self.assertEqual(script_path, config_model.script_command) @@ -43,6 +44,7 @@ def test_create_full_config(self): self.assertEqual(requires_terminal, config_model.requires_terminal) self.assertEqual(bash_formatting, config_model.ansi_enabled) self.assertEqual(output_files, config_model.output_files) + self.assertTrue(config_model.schedulable) def test_create_with_parameter(self): config_model = _create_config_model('conf_p_1', parameters=[create_script_param_config('param1')]) @@ -805,6 +807,43 @@ def test_get_sorted_with_parameters(self): self.assertEqual(expected, config) +class SchedulableConfigTest(unittest.TestCase): + def test_create_with_schedulable_false(self): + config_model = _create_config_model('some-name', config={ + 'scheduling': {'enabled': False}}) + self.assertFalse(config_model.schedulable) + + def test_create_with_schedulable_default(self): + config_model = _create_config_model('some-name', config={}) + self.assertFalse(config_model.schedulable) + + def test_create_with_schedulable_true_and_secure_parameter(self): + config_model = _create_config_model('some-name', config={ + 'scheduling': {'enabled': True}, + 'parameters': [{'name': 'p1', 'secure': True}] + }) + self.assertFalse(config_model.schedulable) + + def test_create_with_schedulable_true_and_included_secure_parameter(self): + config_model = _create_config_model('some-name', config={ + 'scheduling': {'enabled': True}, + 'include': '${p1}', + 'parameters': [{'name': 'p1', 'secure': False}] + }) + another_path = test_utils.write_script_config( + {'parameters': [{'name': 'p2', 'secure': True}]}, + 'another_config') + + self.assertTrue(config_model.schedulable) + + config_model.set_param_value('p1', another_path) + + self.assertFalse(config_model.schedulable) + + def tearDown(self) -> None: + test_utils.cleanup() + + def _create_config_model(name, *, config=None, username=DEF_USERNAME, diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py index b623ba14..887b89b4 100644 --- a/src/tests/test_utils.py +++ b/src/tests/test_utils.py @@ -4,6 +4,8 @@ import stat import threading import uuid +from copy import copy +from unittest.case import TestCase import utils.file_utils as file_utils import utils.os_utils as os_utils @@ -212,7 +214,8 @@ def create_config_model(name, *, parameter_values=None, script_command='ls', output_files=None, - requires_terminal=None): + requires_terminal=None, + schedulable=True): result_config = {} if config: @@ -232,6 +235,9 @@ def create_config_model(name, *, if requires_terminal is not None: result_config['requires_terminal'] = requires_terminal + if schedulable is not None: + result_config['scheduling'] = {'enabled': schedulable} + result_config['script_path'] = script_command return ConfigModel(result_config, path, username, audit_name, parameter_values=parameter_values) @@ -392,6 +398,13 @@ def get_argument(arg_name): return request_handler +def assert_dir_files(expected_files, dir_path, test_case: TestCase): + expected_files_sorted = sorted(copy(expected_files)) + actual_files = sorted(os.listdir(dir_path)) + + test_case.assertSequenceEqual(expected_files_sorted, actual_files) + + class _MockProcessWrapper(ProcessWrapper): def __init__(self, executor, command, working_directory, env_variables): super().__init__(command, working_directory, env_variables) diff --git a/src/tests/web/server_test.py b/src/tests/web/server_test.py index 94e6fae0..f392dd9c 100644 --- a/src/tests/web/server_test.py +++ b/src/tests/web/server_test.py @@ -112,6 +112,7 @@ def start_server(self, port, address): authorizer, None, None, + None, ConfigService(authorizer, self.conf_folder), None, None, diff --git a/src/utils/date_utils.py b/src/utils/date_utils.py index 3b14220f..16f74885 100644 --- a/src/utils/date_utils.py +++ b/src/utils/date_utils.py @@ -1,3 +1,4 @@ +import calendar import sys import time from datetime import datetime, timezone @@ -24,10 +25,6 @@ def sec_to_datetime(time_seconds): return datetime.fromtimestamp(time_seconds, tz=timezone.utc) -def datetime_now(): - return datetime.now(tz=timezone.utc) - - def astimezone(datetime_value, new_timezone): if (datetime_value.tzinfo is not None) or (sys.version_info >= (3, 6)): return datetime_value.astimezone(new_timezone) @@ -47,3 +44,41 @@ def days_to_ms(days): def ms_to_days(ms): return float(ms) / MS_IN_DAY + + +def parse_iso_datetime(date_str): + return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) + + +def to_iso_string(datetime_value: datetime): + if datetime_value.tzinfo is not None: + datetime_value = datetime_value.astimezone(timezone.utc) + + return datetime_value.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def is_past(dt: datetime): + return now(tz=dt.tzinfo) > dt + + +def seconds_between(start: datetime, end: datetime): + delta = end - start + return delta.total_seconds() + + +def add_months(datetime_value: datetime, months): + month = datetime_value.month - 1 + months + year = datetime_value.year + month // 12 + month = month % 12 + 1 + day = min(datetime_value.day, calendar.monthrange(year, month)[1]) + return datetime_value.replace(year=year, month=month, day=day) + + +_mocked_now = None + + +def now(tz=timezone.utc): + if _mocked_now is not None: + return _mocked_now + + return datetime.now(tz) diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py index bc258645..cf9c852b 100644 --- a/src/utils/file_utils.py +++ b/src/utils/file_utils.py @@ -172,9 +172,9 @@ def split_all(path): def to_filename(txt): if os_utils.is_win(): - return txt.replace(':', '-') + return re.sub('[<>:"/\\\\|?*]', '_', txt) - return txt + return txt.replace('/', '_') def create_unique_filename(preferred_path, retries=9999999): diff --git a/src/web/server.py b/src/web/server.py index 7a694e32..4fc75382 100755 --- a/src/web/server.py +++ b/src/web/server.py @@ -35,6 +35,7 @@ from model.parameter_config import WrongParameterUsageException from model.script_config import InvalidValueException, ParameterNotFoundException from model.server_conf import ServerConfig +from scheduling.schedule_service import ScheduleService, UnavailableScriptException, InvalidScheduleException from utils import audit_utils, tornado_utils, os_utils, env_utils from utils import file_utils as file_utils from utils.audit_utils import get_audit_name_from_request @@ -433,8 +434,7 @@ def prepare_download_url(self, file): @tornado.web.stream_request_body -class ScriptExecute(BaseRequestHandler): - +class StreamUploadRequestHandler(BaseRequestHandler): def __init__(self, application, request, **kwargs): super().__init__(application, request, **kwargs) @@ -457,6 +457,11 @@ def prepare(self): def data_received(self, chunk): self.form_reader.read(chunk) + +class ScriptExecute(StreamUploadRequestHandler): + def __init__(self, application, request, **kwargs): + super().__init__(application, request, **kwargs) + @inject_user def post(self, user): script_name = None @@ -800,6 +805,39 @@ def get(self, user, execution_id): self.write(json.dumps(long_log)) +@tornado.web.stream_request_body +class AddSchedule(StreamUploadRequestHandler): + + def __init__(self, application, request, **kwargs): + super().__init__(application, request, **kwargs) + + @inject_user + def post(self, user): + arguments = self.form_reader.values + execution_info = external_model.to_execution_info(arguments) + parameter_values = execution_info.param_values + + if self.form_reader.files: + for key, value in self.form_reader.files.items(): + parameter_values[key] = value.path + + schedule_config = json.loads(parameter_values['__schedule_config']) + del parameter_values['__schedule_config'] + + try: + id = self.application.schedule_service.create_job( + execution_info.script, + parameter_values, + external_model.parse_external_schedule(schedule_config), + user) + except (UnavailableScriptException, InvalidScheduleException) as e: + raise tornado.web.HTTPError(422, reason=str(e)) + except InvalidValueException as e: + raise tornado.web.HTTPError(422, reason=e.get_user_message()) + + self.write(json.dumps({'id': id})) + + def wrap_to_server_event(event_type, data): return json.dumps({ "event": event_type, @@ -863,6 +901,7 @@ def init(server_config: ServerConfig, authenticator, authorizer, execution_service: ExecutionService, + schedule_service: ScheduleService, execution_logging_service: ExecutionLoggingService, config_service: ConfigService, alerts_service: AlertsService, @@ -900,6 +939,7 @@ def init(server_config: ServerConfig, (r'/executions/status/(.*)', GetExecutionStatus), (r'/history/execution_log/short', GetShortHistoryEntriesHandler), (r'/history/execution_log/long/(.*)', GetLongHistoryEntryHandler), + (r'/schedule', AddSchedule), (r'/auth/info', AuthInfoHandler), (r'/result_files/(.*)', DownloadResultFile, @@ -934,6 +974,7 @@ def init(server_config: ServerConfig, application.file_download_feature = file_download_feature application.file_upload_feature = file_upload_feature application.execution_service = execution_service + application.schedule_service = schedule_service application.execution_logging_service = execution_logging_service application.config_service = config_service application.alerts_service = alerts_service diff --git a/web-src/src/assets/css/materializecss/material-datepicker.css b/web-src/src/assets/css/materializecss/material-datepicker.css new file mode 100644 index 00000000..aab0871f --- /dev/null +++ b/web-src/src/assets/css/materializecss/material-datepicker.css @@ -0,0 +1,3 @@ +.input-field.inline .datepicker-container .select-dropdown { + margin-bottom: 0; +} \ No newline at end of file diff --git a/web-src/src/assets/css/materializecss/material-textfield.css b/web-src/src/assets/css/materializecss/material-textfield.css new file mode 100644 index 00000000..9a9791cd --- /dev/null +++ b/web-src/src/assets/css/materializecss/material-textfield.css @@ -0,0 +1,15 @@ +.input-field:after { + content: attr(data-error); + color: #F44336; + font-size: 0.9em; + display: block; + position: absolute; + top: 1.7em; + left: 0.8em; +} + +.input-field input[type="text"]:invalid, +.input-field input[type="number"]:invalid { + border-bottom: 1px solid #e51c23; + box-shadow: 0 1px 0 0 #e51c23; +} diff --git a/web-src/src/common/components/PromisableButton.vue b/web-src/src/common/components/PromisableButton.vue index 5d2ae0c8..6ad508d7 100644 --- a/web-src/src/common/components/PromisableButton.vue +++ b/web-src/src/common/components/PromisableButton.vue @@ -1,23 +1,23 @@ \ No newline at end of file diff --git a/web-src/src/common/components/inputs/TimePicker.vue b/web-src/src/common/components/inputs/TimePicker.vue new file mode 100644 index 00000000..5c300b0a --- /dev/null +++ b/web-src/src/common/components/inputs/TimePicker.vue @@ -0,0 +1,96 @@ + + + + + \ No newline at end of file diff --git a/web-src/src/common/components/textfield.vue b/web-src/src/common/components/textfield.vue index 07faceda..9fcded86 100644 --- a/web-src/src/common/components/textfield.vue +++ b/web-src/src/common/components/textfield.vue @@ -17,6 +17,7 @@ import {isBlankString, isEmptyString, isNull} from '@/common/utils/common'; export default { + name: 'Textfield', props: { 'value': [String, Number], 'config': Object, diff --git a/web-src/src/common/materializecss/color_variables.scss b/web-src/src/common/materializecss/color_variables.scss deleted file mode 100644 index d8a15503..00000000 --- a/web-src/src/common/materializecss/color_variables.scss +++ /dev/null @@ -1,31 +0,0 @@ -@import "materialize-css/sass/components/_color-variables.scss"; - -$light-blue: ( - "darken-1": #039be5 -); - -$green: ( - "base": #4CAF50 -); - -$grey: ( - "base": #9e9e9e, - "lighten-2": #e0e0e0, - "lighten-1": #bdbdbd -); - -$orange: ( - "accent-2": #ffab40 -); - - -// override materialize css colors, because there are too many of them, and they are not used -$colors: ( - "materialize-red": $materialize-red, - "red": $red, - "teal": $teal, - "orange": $orange, - "grey": $grey, - "green": $green, - "light-blue": $light-blue -); \ No newline at end of file diff --git a/web-src/src/common/materializecss/imports/datepicker.js b/web-src/src/common/materializecss/imports/datepicker.js new file mode 100644 index 00000000..b1193cb7 --- /dev/null +++ b/web-src/src/common/materializecss/imports/datepicker.js @@ -0,0 +1,8 @@ +// DO NOT TOUCH ORDER +import './global' +import './modal' +import './select' + +import 'materialize-css/js/datepicker'; +import 'materialize-css/sass/components/_datepicker.scss'; +import '@/assets/css/materializecss/material-datepicker.css' \ No newline at end of file diff --git a/web-src/src/common/materializecss/imports/modal.js b/web-src/src/common/materializecss/imports/modal.js index d304424d..8f772784 100644 --- a/web-src/src/common/materializecss/imports/modal.js +++ b/web-src/src/common/materializecss/imports/modal.js @@ -1,5 +1,6 @@ // DO NOT TOUCH ORDER import './global' +import 'materialize-css/js/anime.min'; import 'materialize-css/js/modal'; import 'materialize-css/sass/components/_modal.scss'; \ No newline at end of file diff --git a/web-src/src/common/style_imports.js b/web-src/src/common/style_imports.js index 362f2443..0bfbc4c4 100644 --- a/web-src/src/common/style_imports.js +++ b/web-src/src/common/style_imports.js @@ -1,10 +1,11 @@ import '@/common/materializecss/imports/global' import 'material-design-icons/iconfont/material-icons.css'; import 'typeface-roboto'; - // DO NOT TOUCH ORDER +import "materialize-css/sass/components/_normalize.scss"; import 'materialize-css/sass/components/_color-classes.scss'; import 'materialize-css/sass/components/_grid.scss'; import 'materialize-css/sass/components/_buttons.scss'; import 'materialize-css/sass/components/_waves.scss'; -import '@/assets/css/materializecss/material-buttons.css'; \ No newline at end of file +import '@/assets/css/materializecss/material-buttons.css'; +import '@/assets/css/materializecss/material-textfield.css'; \ No newline at end of file diff --git a/web-src/src/main-app/components/schedule/SchedulePanel.vue b/web-src/src/main-app/components/schedule/SchedulePanel.vue new file mode 100644 index 00000000..8b82b3b3 --- /dev/null +++ b/web-src/src/main-app/components/schedule/SchedulePanel.vue @@ -0,0 +1,370 @@ + + + + + \ No newline at end of file diff --git a/web-src/src/main-app/components/schedule/ToggleDayButton.vue b/web-src/src/main-app/components/schedule/ToggleDayButton.vue new file mode 100644 index 00000000..df211db9 --- /dev/null +++ b/web-src/src/main-app/components/schedule/ToggleDayButton.vue @@ -0,0 +1,48 @@ + + + + + \ No newline at end of file diff --git a/web-src/src/main-app/components/schedule/schedulePanelFields.js b/web-src/src/main-app/components/schedule/schedulePanelFields.js new file mode 100644 index 00000000..f7031a0d --- /dev/null +++ b/web-src/src/main-app/components/schedule/schedulePanelFields.js @@ -0,0 +1,11 @@ +export const repeatPeriodField = { + required: true, + type: 'int', + min: 1, + max: 50 +}; + +export const repeatTimeUnitField = { + type: 'list', + values: ['hours', 'days', 'weeks', 'months'] +} diff --git a/web-src/src/main-app/components/scripts/ScheduleButton.vue b/web-src/src/main-app/components/scripts/ScheduleButton.vue new file mode 100644 index 00000000..3811622f --- /dev/null +++ b/web-src/src/main-app/components/scripts/ScheduleButton.vue @@ -0,0 +1,74 @@ + + + + + \ No newline at end of file diff --git a/web-src/src/main-app/components/scripts/ScriptViewScheduleHolder.vue b/web-src/src/main-app/components/scripts/ScriptViewScheduleHolder.vue new file mode 100644 index 00000000..807f847e --- /dev/null +++ b/web-src/src/main-app/components/scripts/ScriptViewScheduleHolder.vue @@ -0,0 +1,121 @@ + + + + + \ No newline at end of file diff --git a/web-src/src/main-app/components/scripts/script-parameters-view.vue b/web-src/src/main-app/components/scripts/script-parameters-view.vue index 701b1a09..6d1245f9 100644 --- a/web-src/src/main-app/components/scripts/script-parameters-view.vue +++ b/web-src/src/main-app/components/scripts/script-parameters-view.vue @@ -102,22 +102,6 @@ padding-left: 28px; } - .script-parameters-panel >>> input[type="text"]:invalid, - .script-parameters-panel >>> input[type="number"]:invalid { - border-bottom: 1px solid #e51c23; - box-shadow: 0 1px 0 0 #e51c23; - } - - .script-parameters-panel >>> .input-field:after { - content: attr(data-error); - color: #F44336; - font-size: 0.9rem; - display: block; - position: absolute; - top: 23px; - left: 0.75rem; - } - .script-parameters-panel >>> .input-field .select-wrapper + label { transform: scale(0.8); top: -18px; diff --git a/web-src/src/main-app/components/scripts/script-view.vue b/web-src/src/main-app/components/scripts/script-view.vue index f8aa9b88..d7ca335d 100644 --- a/web-src/src/main-app/components/scripts/script-view.vue +++ b/web-src/src/main-app/components/scripts/script-view.vue @@ -5,7 +5,7 @@
+
+
@@ -27,7 +29,7 @@
  • {{ error }}
    • -
    + @@ -53,11 +59,13 @@ import FileDownloadIcon from '@/assets/file_download.png' import LogPanel from '@/common/components/log_panel' import {deepCloneObject, forEachKeyValue, isEmptyObject, isEmptyString, isNull} from '@/common/utils/common'; + import ScheduleButton from '@/main-app/components/scripts/ScheduleButton'; import ScriptLoadingText from '@/main-app/components/scripts/ScriptLoadingText'; import marked from 'marked'; import {mapActions, mapState} from 'vuex' import {STATUS_DISCONNECTED, STATUS_ERROR, STATUS_EXECUTING, STATUS_FINISHED} from '../../store/scriptExecutor'; import ScriptParametersView from './script-parameters-view' + import ScriptViewScheduleHolder from "@/main-app/components/scripts/ScriptViewScheduleHolder"; export default { data: function () { @@ -67,7 +75,9 @@ errors: [], nextLogIndex: 0, lastInlineImages: {}, - downloadIcon: FileDownloadIcon + downloadIcon: FileDownloadIcon, + scheduleMode: false, + scriptConfigComponentsHeight: 0 } }, @@ -82,13 +92,16 @@ components: { ScriptLoadingText, LogPanel, - ScriptParametersView + ScriptParametersView, + ScheduleButton, + ScriptViewScheduleHolder }, computed: { ...mapState('scriptConfig', { scriptDescription: state => state.scriptConfig ? state.scriptConfig.description : '', - loading: 'loading' + loading: 'loading', + scriptConfig: 'scriptConfig' }), ...mapState('scriptSetup', { parameterErrors: 'errors' @@ -128,6 +141,28 @@ }, enableExecuteButton() { + if (this.scheduleMode) { + return false; + } + + if (this.hideExecutionControls) { + return false; + } + + if (this.loading) { + return false; + } + + if (isNull(this.currentExecutor)) { + return true; + } + + return this.currentExecutor.state.status === STATUS_FINISHED + || this.currentExecutor.state.status === STATUS_DISCONNECTED + || this.currentExecutor.state.status === STATUS_ERROR; + }, + + enableScheduleButton() { if (this.hideExecutionControls) { return false; } @@ -168,7 +203,7 @@ }, showLog() { - return !isNull(this.currentExecutor); + return !isNull(this.currentExecutor) && !this.scheduleMode; }, downloadableFiles() { @@ -209,6 +244,10 @@ killEnabledTimeout() { return isNull(this.currentExecutor) ? null : this.currentExecutor.state.killTimeoutSec; + }, + + schedulable() { + return this.scriptConfig && this.scriptConfig.schedulable; } }, @@ -223,7 +262,7 @@ } }, - executeScript: function () { + validatePreExecution: function () { this.errors = []; const errors = this.parameterErrors; @@ -231,12 +270,29 @@ forEachKeyValue(errors, (paramName, error) => { this.errors.push(paramName + ': ' + error); }); + return false; + } + + return true; + }, + + executeScript: function () { + if (!this.validatePreExecution()) { return; } this.startExecution(); }, + openSchedule: function () { + if (!this.validatePreExecution()) { + return; + } + + this.$refs.scheduleHolder.open(); + this.scheduleMode = true; + }, + ...mapActions('executions', { startExecution: 'startExecution' }), @@ -332,6 +388,25 @@ this.lastInlineImages = deepCloneObject(newValue); } + }, + + scriptConfig: { + immediate: true, + handler() { + this.$nextTick(() => { + // 200 is a rough height for headers,buttons, description, etc. + const otherElemsHeight = 200; + + if (isNull(this.$refs.parametersView)) { + this.scriptConfigComponentsHeight = otherElemsHeight; + return; + } + + const paramHeight = this.$refs.parametersView.$el.clientHeight; + + this.scriptConfigComponentsHeight = paramHeight + otherElemsHeight; + }) + } } } } @@ -366,21 +441,26 @@ } .actions-panel { + margin-top: 8px; display: flex; } - .button-execute { - flex: 6 1 5em; + .actions-panel > .button-gap { + flex: 3 1 1px; + } - margin-right: 0; - margin-top: 6px; + .button-execute { + flex: 4 1 312px; } .button-stop { - flex: 1 0 5em; + margin-left: 16px; + flex: 1 1 104px; + } - margin-left: 12px; - margin-top: 6px; + .schedule-button { + margin-left: 32px; + flex: 1 0 auto; } .script-input-panel { @@ -409,11 +489,16 @@ overflow-y: auto; flex: 1; - margin: 17px 12px 7px; + margin: 20px 0 8px; + } + + .validation-panel .header { + padding-left: 0; } .validation-errors-list { - margin-left: 17px; + margin-left: 12px; + margin-top: 8px; } .validation-errors-list li { diff --git a/web-src/src/main-app/store/index.js b/web-src/src/main-app/store/index.js index f9a30e2c..b8d62f23 100644 --- a/web-src/src/main-app/store/index.js +++ b/web-src/src/main-app/store/index.js @@ -5,6 +5,7 @@ import get from 'lodash/get'; import Vue from 'vue' import Vuex from 'vuex' import authModule from './auth'; +import scheduleModule from './scriptSchedule'; import pageModule from './page'; import scriptConfigModule from './scriptConfig'; @@ -25,7 +26,8 @@ const store = new Vuex.Store({ executions: scriptExecutionManagerModule, auth: authModule, history: historyModule(), - page: pageModule + page: pageModule, + scriptSchedule: scheduleModule }, actions: { init({dispatch}) { diff --git a/web-src/src/main-app/store/mainStoreHelper.js b/web-src/src/main-app/store/mainStoreHelper.js new file mode 100644 index 00000000..600d0fc1 --- /dev/null +++ b/web-src/src/main-app/store/mainStoreHelper.js @@ -0,0 +1,18 @@ +import {forEachKeyValue, isNull} from "@/common/utils/common"; + +export function parametersToFormData(parameterValues) { + const formData = new FormData(); + + forEachKeyValue(parameterValues, function (parameter, value) { + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + const valueElement = value[i]; + formData.append(parameter, valueElement); + } + } else if (!isNull(value)) { + formData.append(parameter, value); + } + }); + + return formData; +} \ No newline at end of file diff --git a/web-src/src/main-app/store/scriptExecutionManager.js b/web-src/src/main-app/store/scriptExecutionManager.js index e25030cb..efff076e 100644 --- a/web-src/src/main-app/store/scriptExecutionManager.js +++ b/web-src/src/main-app/store/scriptExecutionManager.js @@ -3,6 +3,7 @@ import axios from 'axios'; import clone from 'lodash/clone'; import get from 'lodash/get'; import scriptExecutor, {STATUS_EXECUTING, STATUS_FINISHED, STATUS_INITIALIZING} from './scriptExecutor'; +import {parametersToFormData} from "@/main-app/store/mainStoreHelper"; export const axiosInstance = axios.create(); @@ -125,20 +126,9 @@ export default { const parameterValues = clone(rootState.scriptSetup.parameterValues); const scriptName = rootState.scriptConfig.scriptConfig.name; - var formData = new FormData(); + const formData = parametersToFormData(parameterValues); formData.append('__script_name', scriptName); - forEachKeyValue(parameterValues, function (parameter, value) { - if (Array.isArray(value)) { - for (let i = 0; i < value.length; i++) { - const valueElement = value[i]; - formData.append(parameter, valueElement); - } - } else if (!isNull(value)) { - formData.append(parameter, value); - } - }); - const executor = scriptExecutor(null, scriptName, parameterValues); store.registerModule(['executions', 'temp'], executor); store.dispatch('executions/temp/setInitialising'); diff --git a/web-src/src/main-app/store/scriptSchedule.js b/web-src/src/main-app/store/scriptSchedule.js new file mode 100644 index 00000000..b03bf0c7 --- /dev/null +++ b/web-src/src/main-app/store/scriptSchedule.js @@ -0,0 +1,28 @@ +import axios from 'axios'; +import clone from 'lodash/clone'; +import {parametersToFormData} from "@/main-app/store/mainStoreHelper"; + +export const axiosInstance = axios.create(); + +export default { + state: {}, + namespaced: true, + actions: { + schedule({state, commit, dispatch, rootState}, {scheduleSetup}) { + const parameterValues = clone(rootState.scriptSetup.parameterValues); + const scriptName = rootState.scriptConfig.scriptConfig.name; + + const formData = parametersToFormData(parameterValues); + formData.append('__script_name', scriptName); + formData.append('__schedule_config', JSON.stringify(scheduleSetup)) + + return axiosInstance.post('schedule', formData) + .catch(e => { + if (e.response.status === 422) { + e.userMessage = e.response.data; + } + throw e; + }); + }, + } +} diff --git a/web-src/tests/unit/combobox_test.js b/web-src/tests/unit/combobox_test.js index 55eeea1c..faf57c0b 100644 --- a/web-src/tests/unit/combobox_test.js +++ b/web-src/tests/unit/combobox_test.js @@ -34,20 +34,25 @@ describe('Test ComboBox', function () { comboBox.destroy(); }); - function assertListElements(expectedTexts, searchHeader = false) { + function assertListElements(expectedTexts, searchHeader = false, showHeader = true) { const listChildren = comboBox.findAll('li'); - expect(listChildren).toHaveLength(expectedTexts.length + 1); + + const extraChildrenCount = showHeader ? 1 : 0; + + expect(listChildren).toHaveLength(expectedTexts.length + extraChildrenCount); const headerText = listChildren.at(0).text(); if (!searchHeader) { - expect(headerText).toBe('Choose your option'); + if (showHeader) { + expect(headerText).toBe('Choose your option'); + } } else { expect(headerText.trim()).toBe('Search'); } for (let i = 0; i < expectedTexts.length; i++) { const value = expectedTexts[i]; - expect(listChildren.at(i + 1).text()).toBe(value); + expect(listChildren.at(i + extraChildrenCount).text()).toBe(value); } } @@ -123,6 +128,13 @@ describe('Test ComboBox', function () { assertListElements(values); }); + + it('Test hide header', async function () { + comboBox.setProps({showHeader: false}) + await vueTicks(); + + assertListElements(['Value A', 'Value B', 'Value C'], false, false); + }); }); describe('Test values', function () { diff --git a/web-src/tests/unit/common/components/inputs/TimePicker_test.js b/web-src/tests/unit/common/components/inputs/TimePicker_test.js new file mode 100644 index 00000000..552d78da --- /dev/null +++ b/web-src/tests/unit/common/components/inputs/TimePicker_test.js @@ -0,0 +1,138 @@ +'use strict'; + +import {mount} from '@vue/test-utils'; +import TimePicker from "@/common/components/inputs/TimePicker"; +import {vueTicks, wrapVModel} from "../../../test_utils"; + +describe('Test TimePicker', function () { + let timepicker; + + before(function () { + + }); + beforeEach(async function () { + timepicker = mount(TimePicker, { + propsData: { + label: 'Test picker', + value: '15:30', + required: true + } + }); + timepicker.vm.$parent.$forceUpdate(); + wrapVModel(timepicker); + + await vueTicks(); + }); + + afterEach(async function () { + await vueTicks(); + timepicker.destroy(); + }); + + after(function () { + }); + + describe('Test config', function () { + + it('Test initial props', function () { + expect(timepicker.find('label').text()).toBe('Test picker') + expect(timepicker.find('input').element.value).toBe('15:30') + expect(timepicker.vm.value).toBe('15:30') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 19:30', async function () { + timepicker.find('input').setValue('19:30'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('19:30') + expect(timepicker.vm.value).toBe('19:30') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 23:59', async function () { + timepicker.find('input').setValue('23:59'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('23:59') + expect(timepicker.vm.value).toBe('23:59') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 24:00', async function () { + timepicker.find('input').setValue('24:00'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('24:00') + expect(timepicker.vm.value).toBe('15:30') + expect(timepicker.vm.error).toBe('Format HH:MM') + expect(timepicker.currentError).toBe('Format HH:MM') + }); + + it('Test user changes time to 9:45', async function () { + timepicker.find('input').setValue('9:45'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('9:45') + expect(timepicker.vm.value).toBe('9:45') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 2:10', async function () { + timepicker.find('input').setValue('2:10'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('2:10') + expect(timepicker.vm.value).toBe('2:10') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 09:10', async function () { + timepicker.find('input').setValue('09:10'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('09:10') + expect(timepicker.vm.value).toBe('09:10') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test user changes time to 09:60', async function () { + timepicker.find('input').setValue('09:60'); + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('09:60') + expect(timepicker.vm.value).toBe('15:30') + expect(timepicker.vm.error).toBe('Format HH:MM') + expect(timepicker.currentError).toBe('Format HH:MM') + }); + + it('Test system changes time to 16:01', async function () { + timepicker.setProps({'value': '16:01'}) + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('16:01') + expect(timepicker.vm.value).toBe('16:01') + expect(timepicker.vm.error).toBeEmpty() + }); + + it('Test system changes time to 31:01', async function () { + timepicker.setProps({'value': '31:01'}) + + await vueTicks(); + + expect(timepicker.find('input').element.value).toBe('31:01') + expect(timepicker.vm.value).toBe('31:01') + expect(timepicker.vm.error).toBe('Format HH:MM') + expect(timepicker.currentError).toBe('Format HH:MM') + }); + + }); +}); \ No newline at end of file diff --git a/web-src/vue.config.js b/web-src/vue.config.js index 57fb01c5..581adc21 100644 --- a/web-src/vue.config.js +++ b/web-src/vue.config.js @@ -44,7 +44,6 @@ module.exports = { scss: { prependData: '@import "./src/assets/css/color_variables.scss"; ' + '@import "materialize-css/sass/components/_variables.scss"; ' - + '@import "materialize-css/sass/components/_normalize.scss"; ' + '@import "materialize-css/sass/components/_global.scss"; ' + '@import "materialize-css/sass/components/_typography.scss"; ' }